Microsoft/Windows2003-3790
Microsoft/Windows2003-3790
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
50969b0ea2
Windows2003-3790/base/ntos/ke/ia64/trap.s
Microsoft 50969b0ea2 First commit
2020-09-30 16:53:55 +02:00

6482 lines
204 KiB
ArmAsm
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

//++
//
// Module Name:
// trap.s
//
// Abstract:
// Low level interruption handlers
//
// Author:
// Bernard Lint 12-Jun-1995
//
// Environment:
// Kernel mode only
//
// Revision History:
//
//
// Open Design Issues:
//
//--
#if DBG // interruption logging is enabled in checked and free builds
#define INTERRUPTION_LOGGING 1
#endif // DBG
#include "ksia64.h"
.file "trap.s"
.explicit
//
// Globals imported:
//
PublicFunction(KeBugCheckEx)
PublicFunction(KiApcInterrupt)
PublicFunction(KiCheckForSoftwareInterrupt)
PublicFunction(KiDispatchException)
PublicFunction(KiExternalInterruptHandler)
PublicFunction(KiFloatTrap)
PublicFunction(KiFloatFault)
PublicFunction(KiGeneralExceptions)
PublicFunction(KiUnimplementedAddressTrap)
PublicFunction(KiNatExceptions)
PublicFunction(KiMemoryFault)
PublicFunction(KiOtherBreakException)
PublicFunction(KiPanicHandler)
PublicFunction(KiSingleStep)
PublicFunction(KiUnalignedFault)
PublicFunction(PsConvertToGuiThread)
PublicFunction(ExpInterlockedPopEntrySListFault)
PublicFunction(KiDebugFault)
PublicFunction(KeSetLowPsrBit)
PublicFunction(KiIA32ExceptionVectorHandler)
PublicFunction(KiIA32InterruptionVectorHandler)
PublicFunction(KiIA32InterceptionVectorHandler)
.global KiSystemServiceHandler
.global KeServiceDescriptorTableShadow
.global KeGdiFlushUserBatch
.global KdDebuggerEnabled
.global MiDefaultPpe
.global MiNatPte
.global KiIA64RseNumOfMaxDirty
#include "icecap.h"
#ifdef _CAPKERN
PublicFunction(_CAP_ThreadID)
PublicFunction(_CAP_SetCPU)
#endif
// For Conditional Interrupt Logging
#define UserSystemcallBit 61
#define ExternalInterruptBit 62
.global KiVectorLogMask
//
// Register aliases used throughout the entire module
//
//
// Banked general registers
//
// h16-h23 can only be used when psr.ic=1.
//
// h24-h31 can only be used when psr.ic=0 (these are reserved for tlb
// and pal/machine check handlers when psr.ic=1).
//
//
// Shown below are aliases of bank 0 registers used in the low level handlers
// by macros ALLOCATE_TRAP_FRAME, SAVE_INTERRUPTION_RESOURCES, and
// RETURN_FROM_INTERRUPTION. When the code in the macros are changes, these
// register aliases must be reviewed.
//
rHIPSR = h16
rHpT2 = h16
rHIIPA = h17
rHRSC = h17
rHDfhPFS = h17 // used to preserve pfs in KiDisabledFpRegisterVector
rHIIP = h18
rHFPSR = h18
rHRNAT2 = h18
rHOldPreds = h19
rHBrp = h19
rHInitBStore= h19
rHIFS = h20
rHPFS = h20
rHBSP = h20
rHISR = h21
rHUNAT = h21
rHBSPSTORE = h21
rHpT3 = h21
rHSp = h22
rHDfhBrp = h22 // used to preserve brp in KiDisabledFpRegisterVector
rHRNAT = h22
rHpT4 = h22
rHpT1 = h23
rTH3 = h24
rHHandler = h25
rTH1 = h26
rTH2 = h27
rHIIM = h28
rHIFA = h28
rHEPCVa = h29
rHVector = h29
rHEPCVa2 = h30
rPanicCode = h30
rTH4 = h31
//
// General registers used through out module
//
pApc = ps0 // User Apc Pending
pUser = ps1 // mode on entry was user
pKrnl = ps2 // mode on entry was kernel
pUstk = ps3
pKstk = ps4
pEM = ps5 // EM ISA on kernel entry
pIA = ps6 // X86 ISA on kernel entry
pKDbg = ps7 // Kernel debug Active
pUDbg = ps8 // Kernel debug Active
//
// Kernel registers used through out module
//
rkHandler = k6 // specific exception handler
//
// Macro definitions for this module only
//
//
// Define vector/exception entry/exit macros.
// N.B. All HANDLER_ENTRY functions go into .nsc section with
// KiNormalSystemCall being the first.
//
#define HANDLER_ENTRY(Name) \
.##global Name; \
.##proc Name; \
Name::
#define HANDLER_ENTRY_EX(Name, Handler) \
.##global Name; \
.##proc Name; \
.##type Handler, @function; \
.##personality Handler; \
Name::
#define VECTOR_ENTRY(Offset, Name, Extra0) \
.##org Offset; \
.##global Name; \
.##proc Name; \
Name::
#define VECTOR_EXIT(Name) \
.##endp Name
#define HANDLER_EXIT(Name) \
.##endp Name
//++
// Routine:
//
// IO_END_OF_INTERRUPT(rVector,rT1,rT2,pEOI)
//
// Routine Description:
//
// HalEOITable Entry corresponding to the vectorNo is tested.
// If the entry is nonzero, then vectorNo is stored to the location
// specified in the entry. If the entry is zero, return.
//
// Arguements:
//
//
// Notes:
//
// MS preprocessor requires /* */ style comments
//
//--
#define IO_END_OF_INTERRUPT(rVector,rT1,rT2,pEOI) ;\
movl rT1 = KiPcr+PcEOITable ;\
;; ;\
ld8 rT1 = [rT1] ;\
;; ;\
shladd rT2 = rVector,3,rT1 ;\
;; ;\
ld8 rT1 = [rT2] ;\
;; ;\
cmp.ne pEOI = zero, rT1 ;\
;; ;\
(pEOI) st4.rel [rT1] = rVector
//++
// Routine:
//
// VECTOR_CALL_HANDLER(Handler, SpecificHandler)
//
// Routine Description:
//
// Common code for transfer to heavyweight handlers from
// interruption vectors. Put RSE in store intensive mode,
// cover current frame and call handler.
//
// Arguments:
//
// Handler: First level handler for this vector
// SpecificHandler: Specific handler to be called by the generic
// exception handler.
//
// Return Value:
//
// None
//
// Notes:
// Uses just the kernel banked registers (h16-h31)
//
// MS preprocessor requires /* */ style comments
//--
#define VECTOR_CALL_HANDLER(Handler,SpecificHandler) ;\
mov rHIFA = cr##.##ifa ;\
movl rHHandler = SpecificHandler ;\
br##.##sptk Handler ;\
;;
//++
// Routine:
//
// ALLOCATE_TRAP_FRAME
//
// Routine Description:
//
// Common code for allocating trap frame on kernel entry for heavyweight
// handler.
//
// On entry:
//
// On exit: sp -> trap frame; any instruction that depends on sp must be
// placed in the new instruction group. Interruption resources
// ipsr, iipa, iip, predicates, isr, sp, ifs are captured in
// seven of the banked registers h16-23. The last one is used
// by SAVE_INTERRUPTION_STATE as a pointer to save these resources
// in the trap frame.
//
// Return Value:
//
// None
//
// Notes:
// Uses just the kernel banked registers (h16-h31)
//
// MS preprocessor requires /* */ style comments below
//--
#define ALLOCATE_TRAP_FRAME ;\
;\
pOverflow1 = pt2 ;\
pOverflow2 = pt3 ;\
pOverflow3 = pt4 ;\
;\
mov rHIPSR = cr##.##ipsr ;\
movl rTH1 = KiPcr+PcInitialStack ;\
;\
mov rHIIP = cr##.##iip ;\
mov rHOldPreds = pr ;\
cover /* cover and save IFS */;\
;; ;\
;\
ld8 rTH4 = [rTH1], PcBStoreLimit-PcInitialStack ;\
mov rTH3 = ar##.##bsp ;\
tbit##.##z pt1, pt0 = sp, 63 ;\
;; ;\
;\
mov rHIIPA = cr##.##iipa ;\
ld8 rTH2 = [rTH1], PcStackLimit-PcBStoreLimit ;\
mov rHVector = ip /* Save this vectors address */;\
;\
mov rHIFS = cr##.##ifs ;\
mov rHSp = sp ;\
extr##.##u rHpT1 = rHIPSR, PSR_CPL, PSR_CPL_LEN /* get mode */;\
;; ;\
;\
cmp4##.##eq pKrnl, pUser = PL_KERNEL, rHpT1 /* set mode pred */;\
cmp4##.##eq pKstk, pUstk = PL_KERNEL, rHpT1 /* set stack pred */;\
add rHpT1 = PcKernelDebugActive-PcStackLimit, rTH1 ;\
;; ;\
;\
(pKstk) ld8 rTH1 = [rTH1] ;\
(pKstk) cmp##.##geu##.##unc pOverflow2 = rTH3, rTH2 ;\
(pKstk) add sp = -TrapFrameLength, sp /* allocate TF */;\
;; ;\
;\
ld1 rHpT1 = [rHpT1] /* load kernel db state */;\
(pKstk) cmp##.##leu##.##unc pOverflow1 = sp, rTH1 ;\
(pKstk) cmp##.##geu##.##unc pOverflow3 = sp, rTH3 ;\
;; ;\
;\
mov rHISR = cr##.##isr ;\
cmp##.##ne##.##or pKDbg = r0, rHpT1 /* kernel debug active? */;\
mov rPanicCode = PANIC_STACK_SWITCH ;\
;\
(pUstk) add sp = -ThreadStateSaveAreaLength-TrapFrameLength, rTH4 ;\
(pOverflow1) br.spnt.few KiPanicHandler ;\
(pOverflow2) br.spnt.few KiPanicHandler ;\
(pOverflow3) br.spnt.few KiPanicHandler
//++
// Routine:
//
// SAVE_INTERRUPTION_STATE
//
// Routine Description:
//
// Common code for saving interruption state on entry to a heavyweight
// handler.
//
// Arguments:
//
// None
//
// On entry:
//
// sp -> trap frame
//
// On exit:
//
// Static registers gp, teb, sp, fpsr spilled into the trap frame.
// Registers gp, teb, fpsr are set up for kernel mode execution.
//
// Return Value:
//
// None
//
// Notes:
//
// Interruption resources already captured in bank 0 registers h16-h23.
// It's safe to take data TLB fault when saving them into the trap
// frame because kernel stack is always resident in memory. This macro
// is carefully constructed to save the bank registers' contents in
// the trap frame and reuse them to capture other register states as
// soon as they are available. Until we have a virtual register
// allocation scheme in place, the bank 0 register aliases defined at
// the beginning of the file must be updated when this macro is modified.
//
// MS preprocessor requires /* */ style comments below
//--
#define SAVE_INTERRUPTION_STATE ;\
;\
/* Save interruption resources in trap frame */ ;\
;\
;\
ssm (1 << PSR_IC) | (1 << PSR_DFH) | (1 << PSR_AC) ;\
add rHpT1 = TrStIPSR, sp /* -> IPSR */;\
;; ;\
srlz##.##d ;\
st8 [rHpT1] = rHIPSR, TrStISR-TrStIPSR /* save IPSR */;\
add rHpT2 = TrPreds, sp /* -> Preds */;\
;; ;\
;\
st8 [rHpT1] = rHISR, TrStIIP-TrStISR /* save ISR */;\
st8 [rHpT2] = rHOldPreds, TrIntSp-TrPreds ;\
;; ;\
;\
mov rHUNAT = ar##.##unat ;\
st8 [rHpT1] = rHIIP, TrStIFS-TrStIIP /* save IIP */;\
mov rHBrp = brp ;\
;; ;\
;\
mov rHFPSR = ar##.##fpsr ;\
st8 [rHpT1] = rHIFS, TrStIIPA-TrStIFS /* save IFS */;\
mov rHPFS = ar##.##pfs ;\
st8.spill [rHpT2] = rHSp, TrBrRp-TrIntSp /* save SP */;\
;; ;\
;\
st8 [rHpT1] = rHIIPA, TrStFPSR-TrStIIPA /* save IIPA */;\
st8 [rHpT2] = rHBrp, TrRsPFS-TrBrRp ;\
;; ;\
;\
mov rHRSC = ar##.##rsc ;\
st8 [rHpT2] = rHPFS /* save PFS */;\
add rHpT2 = TrApUNAT, sp ;\
;\
mov rHBSP = ar##.##bsp ;\
(pUstk) movl rHInitBStore = KiPcr + PcInitialBStore ;\
;; ;\
;\
mov ar##.##rsc = r0 /* put RSE in lazy mode */;\
st8 [rHpT2] = rHUNAT, TrIntGp-TrApUNAT /* save UNAT */;\
(pUstk) ld8 rHInitBStore = [rHInitBStore] ;\
;; ;\
;\
mov rHBSPSTORE = ar##.##bspstore /* get user bspstore point */;\
st8 [rHpT1] = rHFPSR, TrRsBSP-TrStFPSR /* save FPSR */;\
;; ;\
;\
st8##.##spill [rHpT2] = gp, TrIntTeb-TrIntGp /* spill GP */;\
st8 [rHpT1] = rHBSP, TrRsBSPSTORE - TrRsBSP /* save BSP */;\
(pUstk) mov rHRNAT = ar##.##rnat /* get RNAT */;\
movl rHFPSR = FPSR_FOR_KERNEL /* initial fpsr value */;\
;; ;\
;\
st8 [rHpT1] = rHBSPSTORE /* save BSPSTORE */;\
st8##.##spill [rHpT2] = teb, TrRsRSC-TrIntTeb /* spill TEB (r13) */;\
sub rHBSP = rHBSP, rHBSPSTORE /* size of dirty region */;\
(pUstk) mov teb = kteb /* sanitize teb */;\
(pUstk) dep rHInitBStore = rHBSPSTORE, rHInitBStore, 0, 9 ;\
;; ;\
;\
mov ar##.##fpsr = rHFPSR /* set fpsr */;\
movl rHpT1 = KiPcr + PcKernelGP ;\
;\
(pUstk) mov ar##.##bspstore = rHInitBStore ;\
dep rHRSC = rHBSP, rHRSC, RSC_MBZ1, RSC_LOADRS_LEN ;\
;; ;\
;\
(pUstk) mov ar##.##rnat = rHRNAT /* preserve the user RNAT */;\
st8 [rHpT2] = rHRSC, TrRsRNAT-TrRsRSC ;\
ld8 gp = [rHpT1] /* load GP */;\
;; ;\
;\
/* */;\
/* If previous mode is user, switch to kernel backing store */;\
/* -- uses the "loadrs" approach. Note that we do not save the */;\
/* BSP/BSPSTORE in the trap frame if prvious mode was kernel */;\
/* */;\
;\
(pUstk) mov ar##.##rsc = RSC_KERNEL /* turn rse on, kernel mode */;\
(pUstk) st8 [rHpT2] = rHRNAT /* save user RNAT */;\
/* adjust kernel BSPSTORE */;\
/* for NAT collection */;\
bsw##.##1 /* switch back to user bank */;\
;; /* stop bit required */;\
//++
// Routine:
//
// RETURN_FROM_INTERRUPTION
//
// Routine Description:
//
// Common handler code to restore trap frame and resume execution
// at the interruption address.
//
// Arguments:
//
// Label
//
// Return Value:
//
// None
//
// Note:
//
// On entry: interrrupts disabled, sp -> trap frame
// On exit:
// MS preprocessor requires /* */ style comments below
//--
#define RETURN_FROM_INTERRUPTION(Label) ;\
;\
.##regstk 0,5,2,0 /* must match the alloc instruction below */ ;\
;\
rBSP = loc0 ;\
rRnat = loc1 ;\
;\
rpT1 = t1 ;\
rpT2 = t2 ;\
rpT3 = t3 ;\
rpT4 = t4 ;\
rThread = t6 ;\
rApcFlag = t7 ;\
rT1 = t8 ;\
rT2 = t9 ;\
;\
;\
alloc rT1 = 0,5,2,0 ;\
movl rpT1 = KiPcr + PcCurrentThread /* ->PcCurrentThread */;\
;; ;\
;\
(pUstk) ld8 rThread = [rpT1] ;\
add loc4 = TrStIPSR, sp ;\
(pKstk) br##.##call##.##spnt brp = KiRestoreTrapFrame ;\
;; ;\
;\
ld8 loc4 = [loc4] /* load StIPSR */ ;\
(pUstk) add rpT1 = ThApcState+AsUserApcPending, rThread ;\
(pKstk) br##.##spnt Label##CriticalExitCode ;\
;; ;\
;\
ld1 rApcFlag = [rpT1], ThAlerted-ThApcState-AsUserApcPending ;\
add rBSP = TrRsBSP, sp ;\
add rRnat = TrRsRNAT, sp /* -> user RNAT */;\
;; ;\
;\
st1##.##nta [rpT1] = zero ;\
cmp##.##ne pApc = zero, rApcFlag ;\
(pUstk) tbit##.##nz.unc pUDbg = loc4, PSR_DB /* if non-zero, user debug active */;\
;; ;\
;\
PSET_IRQL (pApc, APC_LEVEL) ;\
movl gp = _gp /* restore to kernel gp value */;\
;\
(pApc) FAST_ENABLE_INTERRUPTS ;\
(pApc) mov out1 = sp ;\
(pApc) br##.##call##.##sptk brp = KiApcInterrupt ;\
;; ;\
;\
ld8 rBSP = [rBSP] /* user BSP */;\
ld8 rRnat = [rRnat] /* user RNAT */;\
;\
(pApc) FAST_DISABLE_INTERRUPTS ;\
PSET_IRQL (pApc, zero) ;\
(pUDbg) br##.##call##.##spnt brp = KiLoadUserDebugRegisters ;\
;; ;\
;\
br##.##call##.##sptk brp = KiRestoreTrapFrame ;\
;; ;\
;\
;\
Label##CriticalExitCode: ;\
;\
add loc2 = TrBrRp, sp ;\
add loc3 = TrRsRSC, sp ;\
bsw##.##0 ;\
;; ;\
;\
ld8 rHBrp = [loc2], TrRsPFS-TrBrRp ;\
ld8 rHRSC = [loc3] ;\
mov loc3 = RSC_KERNEL_DISABLED ;\
;; ;\
;\
invala ;\
(pUstk) movl out0 = KiIA64RseNumOfMaxDirty ;\
;\
mov rHIPSR = loc4 ;\
movl rHpT1 = KiPcr+PcHighFpOwner ;\
;; ;\
;\
ld8 rHPFS = [loc2] ;\
ld8 rHpT4 = [rHpT1], PcCurrentThread-PcHighFpOwner ;\
extr##.##u loc2 = rHRSC, RSC_MBZ1, RSC_LOADRS_LEN ;\
;; ;\
;\
(pUstk) ld8 out0 = [out0] ;\
shr##.##u out1 = loc2, 3 ;\
sub rHBSPSTORE = rBSP, loc2 ;\
;\
ld8 loc4 = [rHpT1], PcKernelDebugActive-PcCurrentThread ;\
(pUstk) mov rHRNAT2 = rRnat ;\
;; ;\
;\
ld1 rHpT1 = [rHpT1] ;\
dep loc3 = loc2, loc3, RSC_LOADRS, RSC_LOADRS_LEN ;\
cmp##.##ne pt2 = rHpT4, loc4 ;\
;; ;\
;\
mov ar##.##rsc = loc3 /* RSE off */ ;\
(pUstk) cmp##.##gt pt0 = out0, out1 ;\
(pt2) dep rHIPSR = 0, rHIPSR, PSR_MFH, 1 ;\
(pUstk) sub loc0 = out0, out1 ;\
(pKstk) br##.##spnt Label##KernelExit ;\
;; ;\
;\
alloc rTH1 = 0, 0, 1, 0 ;\
(pt2) dep rHIPSR = 1, rHIPSR, PSR_DFH, 1 ;\
;; ;\
(pt0) br##.##call##.##sptk brp = KeScrubBackingStoreRegisters ;\
;; ;\
alloc rTH1 = 0,0,0,0 ;\
;; ;\
;\
loadrs /* pull in user regs */;\
;; ;\
;\
Label##KernelExit: ;\
(pUstk) mov ar##.##bspstore = rHBSPSTORE /* restore user BSP */ ;\
dep rHRSC = r0, rHRSC, RSC_MBZ1, RSC_LOADRS_LEN ;\
(pKstk) cmp##.##ne##.##unc pKDbg, pt2 = rHpT1, r0 /* hardware debug? */ ;\
;\
add rHpT4 = TrApUNAT, sp /* -> previous UNAT */;\
mov ar##.##pfs = rHPFS /* restore PFS */;\
add rHIFS = TrStIFS, sp ;\
;; ;\
;\
(pUstk) mov ar##.##rnat = rHRNAT2 /* restore user RNAT */;\
ld8 rHIFS = [rHIFS] /* load IFS */;\
add rHpT1 = TrStIIPA, sp ;\
;; ;\
;\
mov ar##.##rsc = rHRSC /* restore user RSC */;\
ld8 rHUNAT = [rHpT4],TrPreds-TrApUNAT ;\
mov brp = rHBrp ;\
;; ;\
;\
ld8 rHIIPA = [rHpT1], TrStIIP-TrStIIPA ;\
ld8 rHOldPreds = [rHpT4], TrIntSp-TrPreds ;\
(pt2) dep rHIPSR = 0, rHIPSR, PSR_DB, 1 ;\
;; ;\
;\
ld8 rHIIP = [rHpT1], TrStIFS-TrStIIP /* load IIP */;\
ld8##.##fill sp = [rHpT4], TrStIFS-TrIntSp ;\
(pKDbg) dep rHIPSR = 1, rHIPSR, PSR_DB, 1 ;\
;\
rsm 1 << PSR_IC /* reset ic bit */;\
;; ;\
srlz##.##d /* must serialize */;\
;\
/* */;\
/* Restore status registers */;\
/* */;\
;\
mov cr##.##ipsr = rHIPSR /* restore previous IPSR */;\
mov cr##.##iipa = rHIIPA /* restore previous IIPA */;\
;\
mov cr##.##ifs = rHIFS /* restore previous IFS */;\
mov cr##.##iip = rHIIP /* restore previous IIP */;\
mov pr = rHOldPreds, -1 /* restore preds */;\
;; ;\
;\
/* */;\
/* Resume at point of interruption (rfi must be at end of instruction group)*/;\
/* */;\
mov ar##.##unat = rHUNAT /* restore UNAT */;\
mov h17 = r0 /* clear TB loop count */;\
rfi ;\
;;
//
// Interruption Vector Table. First level interruption vectors.
// This section must be 32K aligned. The special section ".drectve"
// is used to pass the align command to the linker.
//
.section .drectve, "MI", "progbits"
string "-section:.ivt,,align=0x8000"
.section .ivt = "ax", "progbits"
KiIvtBase:: // symbol for start of IVT
//++
//
// KiVhptTransVector
//
// Cause: The hardware VHPT walker encountered a TLB miss while attempting to
// reference the virtuall addressed VHPT linear table.
//
// Parameters: cr.iip - address of bundle for which the hardware VHPT walker was
// trying to resolve the TLB miss
//
// cr.ipsr - copy of PSR at the time of the fault
//
// cr.idtr - default translation inforamtion for the address that caused
// a VHPT translation fault
//
// cr.ifa - original faulting address
//
// cr.isr - original faulting status information
//
// Handle: Extracts the PDE index from cr.iha (PTE address in VHPT) and
// generates a PDE address by adding to VHPT_DIRBASE. When accesses
// a page directory entry (PDE), there might be a TLB miss on the
// page directory table and returns a NaT on ld8.s. If so, branches
// to KiPageDirectoryTableFault. If the page-not-present bit of the
// PDE is not set, branches to KiPageNotPresentFault. Otherwise,
// inserts the PDE entry into the data TC (translation cache).
//
//--
VECTOR_ENTRY(0x0000, KiVhptTransVector, cr.ifa)
#if 1
rva = h24
riha = h25
rpr = h26
rPte = h27
rK5 = h28
rPte2 = h28
rCache = h28
rps = h29
risr = h30
riha2 = h31
mov rva = cr.ifa // M0
mov rps = PAGE_SIZE
mov riha = cr.iha // M0
mov rpr = pr // I
mov risr = cr.isr // M0
;;
mov rK5 = ar.k5
tbit.nz pt6 = risr, ISR_SP
;;
#ifndef NO_IHA_CHECK
thash riha2 = rva // M0, for extra IHA checking
#endif
(pt6) cmp.gtu.unc pt6 = rps, rva
(pt6) br.cond.spnt KiPageZeroFault
ld8.s rPte = [riha] // M
tbit.z pt3, pt4 = risr, ISR_X // I
cmp.le pt7,pt8 = 20, rK5
add rK5 = 2, rK5
;;
(pt7) mov ar.k5 = r0
tnat.nz pt0 = rPte // I
(pt8) mov ar.k5 = rK5
tbit.z pt1 = rPte, PTE_VALID // I
(pt0) br.cond.spnt KiPageTableFault // B
(pt1) br.cond.spnt KiPteNotPresentFault // B
extr.u rCache = rPte, 2, 3 // I
;;
cmp.eq pt5 = 1, rCache // A
(pt5) br.cond.spnt KiPageTableFault // B
;;
.pred.rel "mutex",pt3,pt4
(pt4) itc.i rPte // M
;;
(pt3) itc.d rPte // M
;;
#if !defined(NT_UP)
ld8.s rPte2 = [riha] // M
mov rps = PAGE_SHIFT << PS_SHIFT // I
cmp.ne pt0 = zero, zero // I
;;
cmp.ne.or pt0 = rPte2, rPte // M
#ifndef NO_IHA_CHECK
cmp.ne.or pt0 = riha, riha2 // M, check if IHA is correct
#endif
tnat.nz.or pt0 = rPte2 // I
;;
(pt0) ptc.l rva, rps // M
#else
#ifndef NO_IHA_CHECK
cmp.ne pt0 = riha, riha2 // M, check if IHA is correct
mov rps = PAGE_SHIFT << PS_SHIFT // I
;;
(pt0) ptc.l rva, rps // M
#endif
#endif
mov pr = rpr, -1 // I
rfi;; // B
#else
rva = h24
riha = h25
rpr = h26
rpPde = h27
rPde = h28
rPde2 = h29
rps = h30
mov riha = cr.iha // M
mov rva = cr.ifa // M
mov rpr = pr // I
;;
thash rpPde = riha // M
;;
ld8.s rPde = [rpPde] // M, load PDE
;;
tnat.nz pt0, p0 = rPde // I
tbit.z pt1, p0 = rPde, PTE_VALID // I, if non-present page fault
(pt0) br.cond.spnt KiPageDirectoryFault // B
(pt1) br.cond.spnt KiPdeNotPresentFault // B
mov cr.ifa = riha // M
;;
itc.d rPde // M
;;
#if !defined(NT_UP)
ld8.s rPde2 = [rpPde] // M
mov rps = PAGE_SHIFT << PS_SHIFT // I
cmp.ne pt0 = zero, zero // I
;;
cmp.ne.or pt0 = rPde2, rPde // M, if PTEs are different
tnat.nz.or pt0 = rPde2 // I
;;
(pt0) ptc.l riha, rps // M, purge it
#endif
mov pr = rpr, -1 // I
rfi // B
;;
#endif
VECTOR_EXIT(KiVhptTransVector)
//++
//
// KiInstTlbVector
//
// Cause: The VHPT walker aborted the search for the instruction translation.
//
// Parameters: cr.iip - address of bundle for which the hardware VHPT walker was
// trying to resolve the TLB miss
//
// cr.ipsr - copy of PSR at the time of the fault
//
// cr.iha - PTE address in the VHPT which the VHPT walker was attempting to
// reference
//
// cr.iitr - default translation inforamtion for the address that caused
// a instruction TLB miss
//
// cr.isr - faulting status information
//
// Handle: As the VHPT has aborted the search or the implemenation does not
// support the h/w page table walk, the handler needs to emulates the
// function. Since the offending PTE address is already available
// with cr.iha, the handler can access the PTE without performing THASH.
// Accessing a PTE with ld8.s may return a NaT. If so, it branches to
// KiPageTableFault. If the page-not-present bit of the PTE is not set,
// it branches to KiPageFault.
//
// Comments: Merced never casues this fault since it never abort the search on the
// VHPT.
//
//--
VECTOR_ENTRY(0x0400, KiInstTlbVector, cr.iipa)
rva = h24
riha = h25
rpr = h26
rPte = h27
rPte2 = h28
rps = h29
rCache = h28
KiInstTlbVector0:
mov riha = cr.iha // M
mov rva = cr.ifa // M
mov rpr = pr // I
;;
ld8.s rPte = [riha] // M
;;
tnat.nz pt0, p0 = rPte // I
tbit.z pt1, p0 = rPte, PTE_VALID // I
(pt0) br.cond.spnt KiPageTableFault // B
(pt1) br.cond.spnt KiPteNotPresentFault // B
extr.u rCache = rPte, 2, 3 // I
;;
cmp.eq pt3 = 1, rCache // A
(pt3) br.cond.spnt Ki4KInstTlbFault // B
itc.i rPte // M
;;
#if !defined(NT_UP)
ld8.s rPte2 = [riha] // M
mov rps = PAGE_SHIFT << PS_SHIFT // I
cmp.ne pt0 = zero, zero // I
;;
cmp.ne.or pt0 = rPte2, rPte // M
tnat.nz.or pt0 = rPte2 // I
;;
(pt0) ptc.l rva, rps // M
#endif
mov pr = rpr, -1 // I
rfi;; // B
VECTOR_EXIT(KiInstTlbVector)
//++
//
// KiDataTlbVector
//
// Cause: The VHPT walker aborted the search for the data translation.
//
// Parameters: cr.iip - address of bundle for which the hardware VHPT walker was
// trying to resolve the TLB miss
//
// cr.ipsr - copy of PSR at the time of the fault
//
// cr.iha - PTE address in the VHPT which the VHPT walker was attempting to
// reference
//
// cr.idtr - default translation inforamtion for the address that caused
// a data TLB miss
//
// cr.ifa - address that caused a data TLB miss
//
// cr.isr - faulting status information
//
// Handle: As the VHPT has aborted the search or the implemenation does not
// support the h/w page table walk, the handler needs to emulates the
// function. Since the offending PTE address is already available
// with cr.iha, the handler can access the PTE without performing THASH.
// Accessing a PTE with ld8.s may return a NaT. If so, it branches to
// KiPageTableFault. If the page-not-present bit of the PTE is not set,
// it branches to KiPageFault.
//
// Comments: Merced never casues instruction TLB faults since the VHPT search always
// sucesses.
//
//--
VECTOR_ENTRY(0x0800, KiDataTlbVector, cr.ifa)
rva = h24
riha = h25
rpr = h26
rPte = h27
rK5 = h28
rPte2 = h28
rCache = h28
rps = h29
KiDataTlbVector0:
mov riha = cr.iha // M
mov rps = PAGE_SIZE
mov rva = cr.ifa // M
mov rpr = pr // I
;;
mov rK5 = ar.k5
tbit.nz pt6 = risr, ISR_SP
;;
(pt6) cmp.gtu.unc pt6 = rps, rva
(pt6) br.cond.spnt KiPageZeroFault
ld8.s rPte = [riha] // M
cmp.le pt7,pt8 = 20, rK5
add rK5 = 2, rK5
;;
(pt7) mov ar.k5 = r0
(pt8) mov ar.k5 = rK5
extr.u rCache = rPte, 2, 3 // I
;;
cmp.eq pt3 = 1, rCache // A
(pt3) br.cond.spnt Ki4KDataTlbFault // B
tnat.nz pt0, p0 = rPte // I
tbit.z pt1, p0 = rPte, PTE_VALID // I
(pt0) br.cond.spnt KiPageTableFault // B
(pt1) br.cond.spnt KiPteNotPresentFault // B
itc.d rPte // M
;;
#if !defined(NT_UP)
ld8.s rPte2 = [riha] // M
mov rps = PAGE_SHIFT << PS_SHIFT // I
cmp.ne pt0 = zero, zero // I
;;
cmp.ne.or pt0 = rPte2, rPte // M
tnat.nz.or pt0 = rPte2 // I
;;
(pt0) ptc.l rva, rps // M
#endif
mov pr = rpr, -1 // I
rfi;; // B
VECTOR_EXIT(KiDataTlbVector)
//++
//
// KiAltTlbVector
//
// Cause: There was a TLB miss for instruction execution and the VHPT
// walker was not enabled for the referenced region.
//
// Parameters: cr.iip - address of bundle that caused a TLB miss
//
// cr.ipsr - copy of PSR at the time of the fault
//
// cr.idtr - default translation inforamtion for the address that caused
// the fault.
//
// cr.isr - faulting status information
//
// Handle: Currently, NT does not have any use of this vector.
//
//--
VECTOR_ENTRY(0x0c00, KiAltInstTlbVector, cr.iipa)
rva = h24
riha = h25
mov rva = cr.ifa
;;
thash riha = rva
;;
mov cr.iha = riha
;;
srlz.d
br.sptk KiInstTlbVector0
VECTOR_CALL_HANDLER(KiGenericExceptionHandler, KiMemoryFault)
VECTOR_EXIT(KiAltInstTlbVector)
//++
//
// KiAltDataTlbVector
//
// Cause: There was a data TLB miss and the VHPT walker was not enabled for
// the referenced region.
//
// Parameters: cr.iip - address of bundle that caused a TLB miss
//
// cr.ipsr - copy of PSR at the time of the fault
//
// cr.idtr - default translation inforamtion for the address that caused
// the fault.
//
// cr.isr - faulting status information
//
// Handle: Currently, NT does not have any use of this vector.
//
//--
VECTOR_ENTRY(0x1000, KiAltDataTlbVector, cr.ifa)
rva = h24
riha = h25
rpr = h26
rPte = h27
rKseglimit = h28
rIPSR = h30
rISR = h29
rVrn = h31
#if NO_VHPT_WALKER
rRR = h30
mov rva = cr.ifa // M
mov rpr = pr // I
;;
mov rRR = rr[rva]
movl rKseglimit = KSEG3_LIMIT
;;
thash riha = rva
tbit.z pt4 = rRR, 0
mov rIPSR = cr.ipsr
shr.u rVrn = rva, VRN_SHIFT // I, get VPN
(pt4) br.cond.spnt AltFault
;;
mov cr.iha = riha
;;
srlz.d
mov pr = rpr, -1
br.sptk KiDataTlbVector0
;;
AltFault:
#else
mov rva = cr.ifa // M
movl rKseglimit = KSEG3_LIMIT
;;
mov rIPSR = cr.ipsr
mov rpr = pr // I
shr.u rVrn = rva, VRN_SHIFT // I, get VPN
;;
#endif
extr.u rIPSR = rIPSR, PSR_CPL, PSR_CPL_LEN
;;
cmp.ne pt1 = PL_KERNEL, rIPSR
cmp.ne pt2 = KSEG3_VRN, rVrn // M/I
cmp.eq pt4 = KSEG4_VRN, rVrn
(pt1) br.cond.spnt KiCallMemoryFault // if it was User
(pt4) br.cond.spnt KiKseg4Fault
(pt2) br.cond.spnt NoKsegFault // B
cmp.leu pt0 = rKseglimit, rva
(pt0) br.cond.spnt NoKsegFault
mov rISR = cr.isr // M
movl rPte = VALID_KERNEL_PTE // L
mov rIPSR = cr.ipsr // M
shr.u rva = rva, PAGE_SHIFT // I
;;
tbit.z pt2, pt3 = rISR, ISR_SP // I
dep.z rva = rva, PAGE_SHIFT, 32 // I
;;
or rPte = rPte, rva // I
dep rIPSR = 1, rIPSR, PSR_ED, 1 // I
;;
(pt2) itc.d rPte // M
;;
(pt3) mov cr.ipsr = rIPSR // M
;;
mov pr = rpr, -1 // I
rfi // B
;;
NoKsegFault:
rPdeUtbase = h27
rva0 = h29
rPpe = h30
shr.u rva0 = rva, PAGE_SHIFT
movl rPdeUtbase = KiPcr+PcPdeUtbase
;;
dep.z rva0 = rva0, PAGE_SHIFT, VRN_SHIFT-PAGE_SHIFT
ld8 rPdeUtbase = [rPdeUtbase]
movl rPpe = MiDefaultPpe
;;
ld8 rPpe = [rPpe]
cmp.ne pt3, p0 = rva0, rPdeUtbase
(pt3) br.cond.spnt KiPageFault
;;
itc.d rPpe
;;
mov pr = rpr, -1
rfi;;
VECTOR_EXIT(KiAltDataTlbVector)
//++
//
// KiNestedTlbVector
//
// Cause: Instruction/Data TLB miss handler encountered a TLB miss while
// attempting to reference the PTE in the virtuall addressed
// VHPT linear table.
//
// Parameters: cr.iip - address of bundle for which the hardware VHPT walker was
// trying to resolve the TLB miss
//
// cr.ipsr - copy of PSR at the time of VHPT translation fault
//
// cr.iha - address in VHPT which the VHPT walker was attempting to
// reference
//
// cr.idtr - default translation inforamtion for the virtual address
// contained in cr.iha
//
// cr.ifa - original faulting address
//
// cr.isr - faulting status information
//
// h16(riha) - PTE address in the VHPT which caused the Nested miss
//
// Handle: Currently, there is no use for Nested TLB vector. This should be
// a bug fault. Call KiPanicHandler.
//
//--
VECTOR_ENTRY(0x1400, KiNestedTlbVector, cr.ifa)
ALLOCATE_TRAP_FRAME
mov rHIFA = cr.ifa
br.sptk KiPanicHandler
VECTOR_EXIT(KiNestedTlbVector)
//++
//
// KiInstKeyMissVector
//
// Cause: There was a instruction key miss in the translation. Since the
// architecture allows an implementation to choose a unified TC
// structure, the hyper space translation brought by the data
// access-bit may cause a instruction key miss fault. Only erroneous
// user code tries to execute the NT page table and hyper space.
//
// Parameters: cr.iip - address of bundle which caused a instruction key miss fault
//
// cr.ipsr - copy of PSR at the time of the data key miss fault
//
// cr.idtr - default translation inforamtion of the address that caused
// the fault.
//
// cr.isr - faulting status information
//
// Handle: Save the whole register state and call MmAccessFault().
//
//--
VECTOR_ENTRY(0x1800, KiInstKeyMissVector, cr.iipa)
VECTOR_CALL_HANDLER(KiGenericExceptionHandler, KiMemoryFault)
VECTOR_EXIT(KiInstKeyMissVector)
//++
//
// KiDataKeyMissVector
//
// Cause: The referenced translation had the different key ID from the one
// specified the key permission register. This is an indication of
// TLB miss on the NT page table and hyperspace.
//
// Parameters: cr.iip - address of bundle which caused the fault
//
// cr.ipsr - copy of PSR at the time of the data key miss fault
//
// cr.idtr - default translation inforamtion of the address that caused
// the fault.
//
// cr.ifa - address that caused a data key miss
//
// cr.isr - faulting status information
//
// Handle: The handler needs to purge the faulting translation and install
// a new PTE by loading it from the VHPT. The key ID of the IDTR
// for the installing translation should be modified to be the same
// ID as the local region ID. This effectively creates a local
// space within the global kernel space.
//
//--
VECTOR_ENTRY(0x1c00, KiDataKeyMissVector, cr.ifa)
VECTOR_CALL_HANDLER(KiGenericExceptionHandler, KiMemoryFault)
VECTOR_EXIT(KiDataKeyMissVector)
//++
//
// KiDirtyBitVector
//
// Cause: The refereced data translation did not have the dirty-bit set and
// a write operation was made to this page.
//
// Parameters: cr.iip - address of bundle which caused a dirty bit fault
//
// cr.ipsr - copy of PSR at the time of a data access fault
//
// cr.idtr - default translation inforamtion for the address that
// caused the fault
//
// cr.ifa - referenced data address that caused the dirty-bit fault
//
// cr.isr - faulting status information
//
// Handle: Save the whole register state and call MmAccessFault().
//
// Comments: There is always a TLB coherency problem on a multiprocessor
// system. Rather than implementing an atomic operation of setting
// dirty-bit within this handler, the handler instead calls the high
// level C routine, MmAccessFault(), to perform locking the page table
// and setting the dirty-bit of the PTE.
//
// It is too much effort to implement the atomic operation of setting
// the dirty-bit using cmpxchg; a potential nested TLB miss on load/store
// and restoring ar.ccv complicate the design of the handler.
//
//--
VECTOR_ENTRY(0x2000, KiDirtyBitVector, cr.ifa)
VECTOR_CALL_HANDLER(KiGenericExceptionHandler, KiMemoryFault)
VECTOR_EXIT(KiDirtyBitVector)
//++
//
// KiInstAccessBitVector
//
// Cause: There is a access-bit fault on the instruction translation. This only
// happens if the erroneous user mistakenly accesses the NT page table and
// hyper space.
//
// Parameters: cr.iip - address of bundle which caused a instruction access bit fault
//
// cr.ipsr - copy of PSR at the time of a data access fault
//
// cr.idtr - default translation inforamtion for the address that
// caused the fault
//
// cr.ifa - referenced data address that caused the data access-bit fault
//
// cr.isr - faulting status information
//
// Handle: Save the whole register state and call MmAccessFault().
//
//--
VECTOR_ENTRY(0x2400, KiInstAccessBitVector, cr.iipa)
VECTOR_CALL_HANDLER(KiGenericExceptionHandler, KiMemoryFault)
VECTOR_EXIT(KiInstAccessBitVector)
//++
//
// KiDataBitAccessVector
//
// Cause: The reference-bit in the the referenced translation was zero,
// indicating there was a TLB miss on the NT page table or hyperspace.
//
// Parameters: cr.iip - address of bundle which caused a data access bit fault
//
// cr.ipsr - copy of PSR at the time of a data access fault
//
// cr.idtr - default translation inforamtion for the address that
// caused the fault
//
// cr.ifa - referenced data address that caused the data access-bit fault
//
// cr.isr - faulting status information
//
// Handle: The reference-bit is used to fault on PTEs for the NT page table and
// hyperspace. On a data access-bit fault, the handler needs to change the
// the default key ID of the IDTR to be the local key ID. This effectively
// creates the local space within the global kernel space.
//
//--
VECTOR_ENTRY(0x2800, KiDataAccessBitVector, cr.ifa)
rva = h24
rpr = h26
rIPSR = h27
rISR = h31
//
// check to see if non-present fault occurred on a speculative load.
// if so, set IPSR.ed bit. This forces to generate a NaT on ld.s after
// rfi
//
mov rpr = pr
mov rISR = cr.isr // M
mov rIPSR = cr.ipsr // M
;;
tbit.z pt0, p0 = rISR, ISR_SP // I
dep rIPSR = 1, rIPSR, PSR_ED, 1 // I
(pt0) br.cond.spnt KiCallMemoryFault // B
;;
mov cr.ipsr = rIPSR // M
;;
mov pr = rpr, -1 // I
rfi // B
;;
VECTOR_EXIT(KiDataBitAccessVector)
//--------------------------------------------------------------------
// Routine:
//
// KiBreakVector
//
// Description:
//
// Interruption vector for break instruction.
//
// On entry:
//
// IIM contains break immediate value:
// -- BREAK_SYSCALL -> standard system call
// interrupts disabled
// r16-r31 switched to kernel bank
// r16-r31 all available since no TLB faults at this point
//
// Return value:
//
// if system call, sys call return value in v0.
//
// Process:
//--------------------------------------------------------------------
VECTOR_ENTRY(0x2C00, KiBreakVector, cr.iim)
mov rHIIM = cr.iim // get break value
movl rTH1 = KiPcr+PcSavedIIM
;;
st8 [rTH1] = rHIIM
VECTOR_CALL_HANDLER(KiGenericExceptionHandler, KiOtherBreakException)
//
// Do not return from handler
//
VECTOR_EXIT(KiBreakVector)
//--------------------------------------------------------------------
// Routine:
//
// KiExternalInterruptVector
//
// Routine Description:
//
// Interruption vector for External Interrrupt
//
// On entry:
//
// interrupts disabled
// r16-r31 switched to kernel bank
//
// Return value:
//
// none
//
// Process:
//--------------------------------------------------------------------
VECTOR_ENTRY(0x3000, KiExternalInterruptVector, r0)
mov h24 = cr.iip
movl h25 = MM_EPC_VA+0x20
;;
mov h26 = pr
cmp.ne pt0 = h25, h24
add h25 = 0x10, h25
;;
mov h27 = cr.ipsr
(pt0) cmp.ne pt0 = h25, h24
;;
dep h27 = 0, h27, PSR_I, 1
(pt0) br.cond.sptk kei_taken
;;
mov cr.ipsr = h27
;;
mov pr = h26, -1
rfi
;;
kei_taken:
mov pr = h26, -1
;;
ALLOCATE_TRAP_FRAME
;;
SAVE_INTERRUPTION_STATE
br.many KiExternalInterruptHandler
;;
//
// Do not return (rfi from handler)
//
VECTOR_EXIT(KiExternalInterruptVector)
//++
//
// KiPageNotPresentVector
//
// Cause: The translation for the referenced page did not have a present-bit
// set.
//
// Parameters: cr.iip - address of bundle which caused a page not present fault
//
// cr.ipsr - copy of PSR at the time of a page not present ault
//
// cr.idtr - default translation inforamtion for the address that
// caused the fault
//
// cr.ifa - referenced data address if the fault occurred on the data
// reference
//
// cr.isr - faulting status information
//
// Handle: This is the page fault. The handler saves the register context and
// calls MmAccessFault().
//
//--
VECTOR_ENTRY(0x5000, KiPageNotPresentVector, cr.ifa)
rva = h24
riha = h25
rpr = h26
rPte = h27
rps = h29
mov rva = cr.ifa // M
mov rpr = pr // I
;;
thash riha = rva // M
cmp.ne pt1 = r0, r0
mov rps = PAGE_SHIFT << PS_SHIFT // I
;;
ld8.s rPte = [riha] // M
;;
tnat.nz pt0, p0 = rPte // I
tbit.z.or pt1, p0 = rPte, PTE_ACCESS
tbit.z.or pt1, p0 = rPte, PTE_VALID // I, if non-present page fault
(pt0) br.cond.spnt KiPageTableFault // B
(pt1) br.cond.spnt KiPteNotPresentFault // B
//
// if we find a valid PTE that is speculatively fetched into the TLB
// then, purge it and return.
//
ptc.l rva, rps // M
;;
mov pr = rpr, -1 // I
rfi;; // B
VECTOR_EXIT(KiPageNotPresentVector)
//++
//
// KiKeyPermVector
//
// Cause: Read, write or execution key permissions were violated.
//
// Parameters: cr.iip - address of bundle which caused a key permission fault
//
// cr.ipsr - copy of PSR at the time of a key permission fault
//
// cr.idtr - default translation inforamtion for the address that
// caused the fault
//
// cr.ifa - referenced data address if the key permission occurred on
// the data reference
//
// cr.isr - faulting status information
//
// Handle: This should not happen. The EM/NT does not utilize the key permissions.
// The handler saves the register state and calls the bug check.
//
//--
VECTOR_ENTRY(0x5100, KiKeyPermVector, cr.ifa)
VECTOR_CALL_HANDLER(KiGenericExceptionHandler, KiMemoryFault)
VECTOR_EXIT(KiKeyPermVector)
//++
//
// KiInstAccessRightsVector
//
// Cause: The referenced page had a access rights violation.
//
// Parameters: cr.iip - address of bundle which caused a data access bit fault
//
// cr.ipsr - copy of PSR at the time of a data access fault
//
// cr.idtr - default translation inforamtion for the address that
// caused the fault
//
// cr.ifa - referenced data address that caused the data ccess-bit fault
//
// cr.isr - faulting status information
//
// Handle: The handler saves the register context and calls MmAccessFault().
//
//--
VECTOR_ENTRY(0x5200, KiInstAccessRightsVector, cr.iipa)
VECTOR_CALL_HANDLER(KiGenericExceptionHandler, KiMemoryFault)
VECTOR_EXIT(KiInstAccessRightsVector)
//++
//
// KiDataAccessRightsVector
//
// Cause: The referenced page had a data access rights violation.
//
// Parameters: cr.iip - address of bundle which caused a data access rights fault
//
// cr.ipsr - copy of PSR at the time of a data access rights fault
//
// cr.idtr - default translation inforamtion for the address that
// caused the fault
//
// cr.ifa - referenced data address that caused the data access rights
// fault
//
// cr.isr - faulting status information
//
// Handle: The handler saves the register context and calls MmAccessFault().
//
//--
VECTOR_ENTRY(0x5300, KiDataAccessRightsVector, cr.ifa)
rva = h24
rpr = h26
rIPSR = h27
rISR = h31
//
// check to see if non-present fault occurred on a speculative load.
// if so, set IPSR.ed bit. This forces to generate a NaT on ld.s after
// rfi
//
mov rpr = pr
mov rISR = cr.isr // M
mov rIPSR = cr.ipsr // M
;;
tbit.z pt0, p0 = rISR, ISR_SP // I
dep rIPSR = 1, rIPSR, PSR_ED, 1 // I
(pt0) br.cond.spnt KiCallMemoryFault2 // B
;;
mov cr.ipsr = rIPSR // M
;;
mov pr = rpr, -1 // I
rfi // B
;;
KiCallMemoryFault2:
mov rHIFA = cr.ifa
movl rHHandler = KiMemoryFault
mov pr = rpr, -1
br.sptk KiGenericExceptionHandler
VECTOR_EXIT(KiDataAccessRightsVector)
//--------------------------------------------------------------------
// Routine:
//
// KiGeneralExceptionsVector
//
// Description:
//
// Interruption vector for General Exceptions
//
// On entry:
// interrupts disabled
// r16-r31 switched to kernel bank
//
// Return value:
//
// none
//
// Process:
//--------------------------------------------------------------------
VECTOR_ENTRY(0x5400, KiGeneralExceptionsVector, cr.isr)
rPSR_ED = h25
rpr = h26
rIPSR = h27
rCode = h30
rISR = h31
rLfetch = h28
mov rISR = cr.isr // M
movl rPSR_ED = 1 << PSR_ED // L
mov rIPSR = cr.ipsr // M
mov rpr = pr
;;
cmp.ne pt0 = r0, r0
extr.u rCode = rISR, 0, 8
mov rLfetch = ISR_LFETCH|ISR_RESVD_REG << 4
;;
cmp.ne.or pt0 = rCode, rLfetch
tbit.z.or pt0 = rISR, ISR_NA
(pt0) br.cond.sptk KiCallGeneralExceptions
;;
or rIPSR = rIPSR, rPSR_ED // A
;;
mov cr.ipsr = rIPSR // M
mov pr = rpr, -1 // I
;;
rfi
;;
KiCallGeneralExceptions:
mov rHIFA = cr.ifa
movl rHHandler = KiGeneralExceptions
mov pr = rpr, -1
br.sptk KiGenericExceptionHandler
//
// Do not return (rfi from handler)
//
VECTOR_EXIT(KiGeneralExceptionsVector)
//--------------------------------------------------------------------
// Routine:
//
// KiDisabledFpRegisterVector
//
// Description:
//
// Interruption vector for Disabled FP-register vector
//
// On entry:
// interrupts disabled
// r16-r31 switched to kernel bank
//
// Return value:
//
// none
//
// Process:
//--------------------------------------------------------------------
VECTOR_ENTRY(0x5500, KiDisabledFpRegisterVector, cr.isr)
mov rHIPSR = cr.ipsr
mov rHIIP = cr.iip
cover
;;
mov rHIFS = cr.ifs
extr.u rTH1 = rHIPSR, PSR_CPL, PSR_CPL_LEN
mov rHOldPreds = pr
;;
cmp4.eq pKrnl, pUser = PL_KERNEL, rTH1
;;
(pUser) tbit.z.unc pt0, pt1 = rHIPSR, PSR_DFH // if dfh not set,
// dfl must be set
(pKrnl) br.spnt.few Kdfrv10 // Kernel mode should never get here.
;;
(pt1) ssm 1 << PSR_IC // set ic bit
(pt1) mov rHDfhPFS = ar.pfs
(pt1) mov rHDfhBrp = brp
;;
(pt1) srlz.d
(pt1) br.call.sptk.many brp = KiRestoreHigherFPVolatile
(pt0) br.spnt.few Kdfrv10
;;
rsm 1 << PSR_IC // reset ic bit
dep rHIPSR = 0, rHIPSR, PSR_DFH, 1 // reset dfh bit
mov brp = rHDfhBrp
;;
srlz.d
mov cr.ifs = rHIFS
mov ar.pfs = rHDfhPFS
mov cr.ipsr = rHIPSR
mov cr.iip = rHIIP
mov pr = rHOldPreds, -1
;;
rfi
;;
Kdfrv10:
mov pr = rHOldPreds, -1
VECTOR_CALL_HANDLER(KiGenericExceptionHandler, KiGeneralExceptions)
;;
VECTOR_EXIT(KiDisabledFpRegisterVector)
//--------------------------------------------------------------------
// Routine:
//
// KiNatConsumptionVector
//
// Description:
//
// Interruption vector for Nat Consumption Vector
//
// On entry:
// interrupts disabled
// r16-r31 switched to kernel bank
//
// Return value:
//
// none
//
// Process:
//--------------------------------------------------------------------
VECTOR_ENTRY(0x5600, KiNatConsumptionVector, cr.isr)
mov h16 = cr.isr
mov h17 = pr
;;
mov rHIFA = cr.ifa
extr.u h16 = h16, 4, 4 // extract the error code
;;
cmp.eq pt0, pt1 = ISR_NAT_PAGE, h16
;;
(pt0) movl rHHandler = KiMemoryFault
(pt1) movl rHHandler = KiNatExceptions
;;
mov pr = h17, -1
br.sptk KiGenericExceptionHandler
VECTOR_EXIT(KiNatConsumptionVector)
//++
//
// KiSpeculationVector
//
// Cause: CHK.S, CHK.A, FCHK detected an exception condition.
//
// Parameters: cr.iip - address of bundle which caused a speculation fault
//
// cr.ipsr - copy of PSR at the time of a speculation fault
//
// cr.iipa - address of bundle containing the last
// successfully executed instruction
//
// cr.iim - contains the immediate value in either
// CHK.S, CHK.A, or FCHK opecode
//
// cr.isr - faulting status information
//
// Handle: The handler implements a branch operation to the
// recovery code specified by the IIM IP-offset.
//
// Note: This code will not be exercised until the compiler
// generates the speculation code.
//
// TBD: Need to check for taken branch trap.
//
//--
VECTOR_ENTRY(0x5700, KiSpeculationVector, cr.iim)
mov h16 = cr.iim // get imm offset
mov h17 = cr.iip // get IIP
;;
extr h16 = h16, 0, 21 // get sign-extended
mov h18 = cr.ipsr
;;
shladd h16 = h16, 4, h17 // get addr for recovery handler
dep h18 = 0, h18, PSR_RI, 2 // zero target slot number
;;
mov cr.ipsr = h18
mov cr.iip = h16
;;
rfi
;;
VECTOR_EXIT(KiSpeculationVector)
//++
//
// KiDebugFaultVector
//
// Cause: A unaligned data access fault has occured
//
// Parameters: cr.iip - address of bundle causing the fault.
//
// cr.ipsr - copy of PSR at the time of interruption.
//
// cr.iipa - address of bundle containing the last
// successfully executed instruction
//
// cr.isr - faulting status information. ISR.ei bits are
// set to indicate which instruction caused the
// exception.
// The ISR.code contains information about the
// FP exception fault. See trapc.c and the EAS
//
//--
VECTOR_ENTRY(0x5900, KiDebugFaultVector, cr.isr)
VECTOR_CALL_HANDLER(KiGenericExceptionHandler, KiDebugFault)
//
// Do not return (rfi from handler)
//
VECTOR_EXIT(KiDebugFaultVector)
//++
//
// KiUnalignedFaultVector
//
// Cause: A unaligned data access fault has occured
//
// Parameters: cr.iip - address of bundle causing the fault.
//
// cr.ipsr - copy of PSR at the time of interruption.
//
// cr.iipa - address of bundle containing the last
// successfully executed instruction
//
// cr.isr - faulting status information. ISR.ei bits are
// set to indicate which instruction caused the
// exception.
// The ISR.code contains information about the
// FP exception fault. See trapc.c and the EAS
//
//--
VECTOR_ENTRY(0x5a00, KiUnalignedFaultVector, cr.ifa)
VECTOR_CALL_HANDLER(KiGenericExceptionHandler, KiUnalignedFault)
//
// Do not return (rfi from handler)
//
VECTOR_EXIT(KiUnalignedFaultVector)
//++
//
// KiFloatFaultVector
//
// Cause: A floating point fault has occured
//
// Parameters: cr.iip - address of bundle causing the fault.
//
// cr.ipsr - copy of PSR at the time of interruption.
//
// cr.iipa - address of bundle containing the last
// successfully executed instruction
//
// cr.isr - faulting status information. ISR.ei bits are
// set to indicate which instruction caused the
// exception.
// The ISR.code contains information about the
// FP exception fault. See trapc.c and the EAS
//
//--
VECTOR_ENTRY(0x5c00, KiFloatFaultVector, cr.isr)
VECTOR_CALL_HANDLER(KiGenericExceptionHandler, KiFloatFault)
//
// Do not return (rfi from handler)
//
VECTOR_EXIT(KiFloatFaultVector)
//++
//
// KiFloatTrapVector
//
// Cause: A floating point trap has occured
//
// Parameters: cr.iip - address of bundle with the instruction to be
// executed next.
//
// cr.ipsr - copy of PSR at the time of interruption.
//
// cr.iipa - address of bundle containing the last
// successfully executed instruction
//
// cr.isr - faulting status information. ISR.ei bits are
// set to indicate which instruction caused the
// exception.
// The ISR.code contains information about the
// FP trap. See trapc.c and the EAS
//
//--
VECTOR_ENTRY(0x5d00, KiFloatTrapVector, cr.isr)
VECTOR_CALL_HANDLER(KiGenericExceptionHandler, KiFloatTrap)
//
// Do not return (rfi from handler)
//
VECTOR_EXIT(KiFloatTrapVector)
//++
//
// KiLowerPrivilegeVector
//
// Cause: A branch lowers the privilege level and PSR.lp is 1.
// Or an attempt made to execute an instruction
// in the unimplemented address space.
// This trap is higher priority than taken branch
// or single step traps.
//
// Parameters: cr.iip - address of bundle containing the instruction to
// be executed next.
//
// cr.ipsr - copy of PSR at the time of interruption.
//
// cr.iipa - address of bundle containing the last
// successfully executed instruction
//
// cr.isr - faulting status information. The ISR.code
// contains a bit vector for all traps which
// occurred in the trapping bundle.
//
//--
VECTOR_ENTRY(0x5e00, KiLowerPrivilegeVector, cr.iipa)
mov rHISR = cr.isr
mov rHOldPreds = pr
mov rHIIPA = cr.iipa
mov rHIPSR = cr.ipsr
;;
tbit.z pt1, pt0 = rHISR, ISR_UI_TRAP
(pt1) br.cond.spnt Klpv10
;;
mov rHIFA = cr.iip
movl rHHandler = KiUnimplementedAddressTrap
mov pr = rHOldPreds, -2 // must restore predicates
br.sptk KiGenericExceptionHandler
;;
Klpv10:
mov rHVector = ip
mov rPanicCode = UNEXPECTED_KERNEL_MODE_TRAP
br.sptk KiPanicHandler
VECTOR_EXIT(KiLowerPrivilegeVector)
//++
//
// KiTakenBranchVector
//
// Cause: A taken branch was successfully execcuted and the PSR.tb
// bit is 1. This trap is higher priority than single step trap.
//
// Parameters: cr.iip - address of bundle containing the instruction to
// be executed next.
//
// cr.ipsr - copy of PSR at the time of interruption.
//
// cr.iipa - address of bundle containing the last
// successfully executed instruction
//
// cr.isr - faulting status information. The ISR.code
// contains a bit vector for all traps which
// occurred in the trapping bundle.
//
//--
VECTOR_ENTRY(0x5f00, KiTakenBranchVector, cr.iipa)
mov rHIPSR = cr.ipsr
movl rHEPCVa = MM_EPC_VA+0x20 // user system call entry point
mov rHIIP = cr.iip
movl rHpT1 = KiPcr+PcInitialStack
;;
ld8 rHpT1 = [rHpT1]
extr.u rTH1 = rHIPSR, PSR_CPL, PSR_CPL_LEN
mov rHOldPreds = pr
;;
cmp.eq pt0 = rHEPCVa, rHIIP
;;
(pt0) ssm 1 << PSR_IC
(pt0) movl rHpT3 = 1 << PSR_LP
;;
(pt0) or rHpT3 = rHIPSR, rHpT3
(pt0) srlz.d
add rHpT1=-ThreadStateSaveAreaLength-TrapFrameLength+TrStIPSR,rHpT1
(pt0) br.spnt.few Ktbv10
mov pr = rHOldPreds, -2
VECTOR_CALL_HANDLER(KiGenericExceptionHandler, KiSingleStep)
;;
Ktbv10:
st8 [rHpT1] = rHpT3
movl rHpT3 = 1 << PSR_SS | 1 << PSR_TB | 1 << PSR_DB
;;
rsm 1 << PSR_IC
mov pr = rHOldPreds, -2
andcm rHIPSR = rHIPSR, rHpT3 // clear ss, tb, db bits
;;
srlz.d
mov cr.ipsr = rHIPSR
;;
rfi
;;
VECTOR_EXIT(KiTakenBranchVector)
//++
//
// KiSingleStepVector
//
// Cause: An instruction was successfully execcuted and the PSR.ss
// bit is 1.
//
// Parameters: cr.iip - address of bundle containing the instruction to
// be executed next.
//
// cr.ipsr - copy of PSR at the time of interruption.
//
// cr.iipa - address of bundle containing the last
// successfully executed instruction
//
// cr.isr - faulting status information. The ISR.code
// contains a bit vector for all traps which
// occurred in the trapping bundle.
//
//--
VECTOR_ENTRY(0x6000, KiSingleStepVector, cr.iipa)
mov rHIPSR = cr.ipsr
movl rHEPCVa = MM_EPC_VA+0x20 // user system call entry point
mov rHIIP = cr.iip
movl rHpT1 = KiPcr+PcInitialStack
;;
ld8 rHpT1 = [rHpT1]
extr.u rTH1 = rHIPSR, PSR_CPL, PSR_CPL_LEN
mov rHOldPreds = pr
;;
cmp.eq pt0 = rHEPCVa, rHIIP
;;
(pt0) ssm 1 << PSR_IC
(pt0) movl rHpT3 = 1 << PSR_LP
;;
(pt0) or rHpT3 = rHIPSR, rHpT3
(pt0) srlz.d
add rHpT1=-ThreadStateSaveAreaLength-TrapFrameLength+TrStIPSR,rHpT1
(pt0) br.spnt.few Kssv10
mov pr = rHOldPreds, -2
VECTOR_CALL_HANDLER(KiGenericExceptionHandler, KiSingleStep)
;;
Kssv10:
st8 [rHpT1] = rHpT3
movl rHpT3 = 1 << PSR_SS | 1 << PSR_DB
;;
rsm 1 << PSR_IC
mov pr = rHOldPreds, -2
andcm rHIPSR = rHIPSR, rHpT3 // clear ss, db bits
;;
srlz.d
mov cr.ipsr = rHIPSR
;;
rfi
;;
VECTOR_EXIT(KiSingleStepVector)
//++
//
// KiIA32ExceptionVector
//
// Cause: A fault or trap was generated while executing from the
// iA-32 instruction set.
//
// Parameters: cr.iip - address of the iA-32 instruction causing interruption
//
// cr.ipsr - copy of PSR at the time of the instruction
//
// cr.iipa - Address of the last successfully executed
// iA-32 or EM instruction
//
// cr.isr - The ISR.ei exception indicator is cleared.
// ISR.iA_vector contains the iA-32 interruption vector
// number. ISR.code contains the iA-32 16-bit error cod
//
// Handle: Save the whole register state and
// call KiIA32ExceptionVectorHandler()().
//
//--
VECTOR_ENTRY(0x6900, KiIA32ExceptionVector, r0)
mov rHIIM = cr.iim // save info from IIM
movl rTH1 = KiPcr+PcSavedIIM
;;
st8 [rTH1] = rHIIM
VECTOR_CALL_HANDLER(KiGenericExceptionHandler,
KiIA32ExceptionVectorHandler)
VECTOR_EXIT(KiIA32ExceptionVector)
//++
//
// KiIA32InterceptionVector
//
// Cause: A interception fault or trap was generated while executing
// from the iA-32 instruction set.
//
// Parameters: cr.iip - address of the iA-32 instruction causing interruption
//
// cr.ipsr - copy of PSR at the time of the instruction
//
// cr.iipa - Address of the last successfully executed
// iA-32 or EM instruction
//
// cr.isr - The ISR.ei exception indicator is cleared.
// ISR.iA_vector contains the iA-32 interruption vector
// number. ISR.code contains the iA-32specific
// interception information
//
// Handle: Save the whole register state and
// call KiIA32InterceptionVectorHandler()().
//
//--
VECTOR_ENTRY(0x6a00, KiIA32InterceptionVector, r0)
mov rHIIM = cr.iim // save info from IIM
movl rTH1 = KiPcr+PcSavedIIM
;;
st8 [rTH1] = rHIIM
VECTOR_CALL_HANDLER(KiGenericExceptionHandler,
KiIA32InterceptionVectorHandler)
VECTOR_EXIT(KiIA32InterceptionVector)
//++
//
// KiIA32InterruptionVector
//
// Cause: An iA software interrupt was executed
//
// Parameters: cr.iip - address of the iA-32 instruction causing interruption
//
// cr.ipsr - copy of PSR at the time of the instruction
//
// cr.iipa - Address of the last successfully executed
// iA-32 or EM instruction
//
// cr.isr - ISR.iA_vector contains the iA-32 defined vector
// number. ISR.code contains 0
// ISR.ei excepting instruction indicator is cleared.
// ISR.iA_vector contains the iA-32 instruction vector.
// ISR.code contains iA-32 specific information.
//
// Handle: Save the whole register state and
// call KiIA32InterruptionVectorHandler()().
//
//--
VECTOR_ENTRY(0x6b00, KiIA32InterruptionVector, r0)
// This one doesn't need IIM, so we won't bother to save it
VECTOR_CALL_HANDLER(KiGenericExceptionHandler,
KiIA32InterruptionVectorHandler)
VECTOR_EXIT(KiIA32InterruptionVector)
//
// All non-VECTOR_ENTRY functions must follow KiNormalSystemCall.
//
// N.B. KiNormalSystemCall must be the first function body in the .nsc
// section.
//
//--------------------------------------------------------------------
// Routine:
//
// KiNormalSystemCall
//
// Description:
//
// Handler for normal (not fast) system calls
//
// On entry:
//
// ic off
// interrupts disabled
// v0: contains sys call #
// cover done by call
// r32-r39: sys call arguments
// CFM: sof = # args, ins = 0, outs = # args
// clear mfh bit (high fp registers are scratch per s/w convention)
//
// Return value:
//
// v0: system call return value
//
// Process:
//
//--------------------------------------------------------------------
.section .drectve, "MI", "progbits"
string " -section:.nsc,,align=0x2000"
.section .nsc = "ax", "progbits"
HANDLER_ENTRY_EX(KiNormalSystemCall, KiSystemServiceHandler)
.prologue
.unwabi @nt, EXCEPTION_FRAME
rPFS = t0
rThread = t1 // current thread
rIFS = t1
rIIP = t2
rPreds = t3
rIPSR = t4
rUNAT = t5
rSp = t6
rpT1 = t7
rpT2 = t8
rpT3 = t9
rpT4 = t10
rT0 = t11
rT1 = t12
rT2 = t13
rT3 = t14
rT4 = t15
rKDbgActive = t16
rIntNats = t17
rpSd = t16 /* -> service descriptor entry */
rArgTable = t18 /* pointer to argument table */
rArgNum = t20 /* number of arguments */
rArgBytes = t21
rRscD = t16
rRNAT = t17
rRscE = t18
rKBSPStore = t18
rBSPStore = t19
rRscDelta = t20
rBSP = t21
rPreviousMode = t22
pInvl = ps9 /* pInvl = not GUI service */
pVal = pt1
pGui = pt2 /* true if GUI call */
pNoGui = pt3 /* true if no GUI call */
pNatedArg = pt4 /* true if any input argument */
/* register is Nat'ed */
pNoCopy = pt5 /* no in-memory arguments to copy */
pCopy = pt6
pNatedSp = pt7
mov rUNAT = ar.unat
mov rPreds = pr
mov rPreviousMode = KernelMode
mov rIPSR = psr
rsm 1 << PSR_I | 1 << PSR_MFH
br.sptk Knsc_Allocate
;;
//
// N.B. KiUserSystemCall is at an offset of 0x20 from KiNormalSystemCall.
// Whenever this offset is changed, the definition of kernel system call
// stub in services.stb must be updated to reflect the new value.
//
ALTERNATE_ENTRY(KiUserSystemCall)
mov rUNAT = ar.unat
mov rPreds = pr
epc
;;
mov rIPSR = psr
rsm (1 << PSR_I) | (1 << PSR_BE)
mov rPreviousMode = UserMode
;; // stop bit needed to ensure interrupt is off
Knsc_Allocate::
#if defined(INTERRUPTION_LOGGING)
// For Conditional Interrupt Logging
movl rpT1 = KiVectorLogMask
;;
ld8 rT1 = [rpT1]
;;
tbit.z pt1 = rT1, UserSystemcallBit
(pt1) br.cond.sptk EndOfLogging1
mov rT1 = 0x80 // dummy offset for sys call
movl rpT1 = KiPcr+PcInterruptionCount
mov rT2 = b0
mov rT3 = MAX_NUMBER_OF_IHISTORY_RECORDS - 1
;;
ld4.nt1 rT4 = [rpT1] // get current count
or rT1 = rT1, rPreviousMode // kernel/user
;;
add rT0 = 1, rT4 // incr count
and rT4 = rT3, rT4 // index of current entry
add rpT2 = 0x1000-PcInterruptionCount, rpT1 // base of history
;;
st4.nta [rpT1] = rT0 // save count
shl rT4 = rT4, 5 // offset of current entry
;;
add rpT2 = rpT2, rT4 // address of current entry
;;
st8 [rpT2] = rT1, 8 // save sys call offset
;;
st8 [rpT2] = rT2, 8 // save return address
;;
st8 [rpT2] = rIPSR, 8 // save psr
;;
st8.spill [rpT2] = v0; // save sys call number
;;
// For Conditional Interrupt Logging
EndOfLogging1:
#endif // INTERRUPTION_LOGGING
//
// if sp is Nat'ed return to caller with an error status
// N.B. sp is spilled and a value of zero is saved in the IntNats field
//
tnat.nz pNatedSp = sp
movl rpT4 = KiPcr+PcInitialStack
;;
mov rSp = sp
(pNatedSp) movl rT4 = Knsc_ErrorReturn
;;
tnat.nz pNatedArg = v0 // Test service number for NAT.
cmp.eq pUser, pKrnl = UserMode, rPreviousMode
mov rBSP = ar.bsp
(pNatedSp) movl v0 = STATUS_IA64_INVALID_STACK
;;
(pUser) ld8 sp = [rpT4], PcInitialBStore-PcInitialStack // set new sp
mov rIIP = brp
mov rPFS = ar.pfs
;;
mov rT2 = ar.rsc
(pUser) ld8 rKBSPStore = [rpT4], PcKernelDebugActive-PcInitialBStore
(pNatedSp) mov bt1 = rT4
;;
(pUser) ld1 rKDbgActive = [rpT4], PcCurrentThread-PcKernelDebugActive
(pKrnl) add rpT4 = PcCurrentThread-PcInitialStack, rpT4
extr.u rIFS = rPFS, 0, 38
mov rT0 = ar.fpsr
(pUser) movl rT3 = 1 << PSR_SS | 1 << PSR_DB | 1 << PSR_TB | 1 << PSR_LP
(pUser) mov ar.rsc = r0 // put RSE in lazy mode
(pUser) add sp = -ThreadStateSaveAreaLength-TrapFrameLength, sp
(pKrnl) add sp = -TrapFrameLength, sp // allocate TF
;;
(pUser) mov rBSPStore = ar.bspstore // get user bsp store point
add rpT1 = TrStIPSR, sp // -> IPSR
add rpT2 = TrIntSp, sp // -> IntSp
;;
//
// Get the previous value of the IPRS from the stack. This is used to get user's debug
// values TB, SS, and DB from before they call the system service. KiSingleStepVector and
// KiTakenBranchVector clear these bits if they hit on the EPC instruction. This value is
// initialized to zero at thread startup and then copied if the user trap frame moves because
// of stack convertion or user call backs.
//
(pUser) ld8 rT4 = [rpT1]
st8.spill [rpT2] = rSp, TrApUNAT-TrIntSp // sp is not Nat'ed
add rpT3 = TrStIFS, sp // -> IFS
(pNatedArg) mov v0 = SERVICE_NUMBER_MASK // v0 is NATed set it to the maximum system service number.
dep rIFS = 1, rIFS, 63, 1 // set IFS.v
;;
st8 [rpT2] = rUNAT, TrPreds-TrApUNAT
st8 [rpT3] = rIFS, TrRsPFS-TrStIFS // save IFS
(pUser) and rT4 = rT3, rT4 // capture psr.db, tb, ss
;;
st8 [rpT2] = rPreds, TrIntNats-TrPreds
st8.nta [rpT3] = rPFS, TrStFPSR-TrRsPFS // save PFS
(pUser) mov rT3 = rIPSR
rBspOff = t5
extr.u rBspOff = rBSP, 0, 9
extr.u t0 = rIFS, 0, 7
extr.u rIFS = rIFS, 7, 7
;;
(pUser) mov rRNAT = ar.rnat
movl rT1 = 1 << PSR_I | 1 << PSR_MFH | 1 << PSR_BE
sub rIFS = t0, rIFS
movl t0 = 1 << PSR_I | 1 << PSR_BN
;;
(pUser) or rIPSR = rIPSR, rT4
(pUser) cmp.ne.unc pKDbg, pt2 = zero, rKDbgActive // kernel debug active?
shladd rBspOff = rIFS, 3, rBspOff
shladd rBSP = rIFS, 3, rBSP
mov rT4 = 0x1F8
(pUser) dep t0 = 1, t0, PSR_CPL, 2
;;
st8 [rpT2] = zero,TrIntGp-TrIntNats // all integer Nats are 0
st8 [rpT3] = rT0, TrStIIP-TrStFPSR // save FPSR
(pUser) andcm rT3 = rT3, rT1 // clear i, mfh, be
cmp.ge pt0 = rBspOff, rT4
or rIPSR = t0, rIPSR // set i, cpl, bn for the saved IPSR
;;
st8 [rpT1] = rIPSR // save IPSR
st8 [rpT3] = rIIP, TrBrRp-TrStIIP // save IIP
(pt2) dep rT3 = 0, rT3, PSR_DB, 1 // disable db in kernel
;;
st8.nta [rpT3] = rIIP, TrRsBSP-TrBrRp // save BRP
st8.spill [rpT2] = gp, TrIntV0-TrIntGp // Save GP even though it is a temporary
// If someone does a get/set context and GP
// is zero in the trap fram then set context
// will think IIP is a plabel and dereference i.
(pt0) add rBSP = 8, rBSP
(pUser) dep rT3 = 1, rT3, PSR_AC, 1 // enable alignment check
;;
(pt0) add rIFS = 1, rIFS
st8.nta [rpT2] = v0, TrRsRSC-TrIntV0 // Sanitize the return value in trap frame.
st8 [rpT3] = rBSP, TrRsBSPSTORE-TrRsBSP
(pUser) sub rBSP = rBSP, rBSPStore
(pUser) mov psr.l = rT3
;;
(pUser) st8 [rpT3] = rBSPStore // save user BSP Store
(pUser) dep rT2 = rBSP, rT2, RSC_MBZ1, RSC_LOADRS_LEN
(pUser) mov teb = kteb // get Teb pointer
movl rT1 = Knsc10 // Leave the EPC page
;;
st8 [rpT2] = rT2, TrRsRNAT-TrRsRSC // save RSC
(pUser) dep rKBSPStore = rBSPStore, rKBSPStore, 0, 9
mov bt0 = rT1 // Addres of Knsc10 in KSEG0.
;;
(pUser) mov ar.bspstore = rKBSPStore // switch to kernel BSP
movl gp = _gp // set up kernel gp
;;
(pUser) mov ar.rnat = rRNAT // preserve uer RNAT bits
(pUser) st8 [rpT2] = rRNAT // save RNAT in trap frame
(pUser) mov ar.rsc = RSC_KERNEL // turn rse on, in kernel mode
(pNatedSp) br.spnt bt1
br.sptk bt0
;;
//
// Register aliases for rest of procedure
//
#ifdef _CAPKERN
.regstk 8, 9, 2, 0
#else
.regstk 8, 9, 0, 0
#endif
rScallGp = loc0
rpThObj = loc1 // pointer to thread object
rSavedV0 = loc2 // saved v0 for GUI thread
rpEntry = loc3 // syscall routine entry point
rSnumber = loc4 // service number
rSdOffset = loc5
rCount = loc6 /* index of the first Nat'ed input register */
rUserSp = loc7
rUserPFS = loc8
//
// The following code is an out of band error path. Since we always
// branch between the EPC page to Knsc10 below.
//
Knsc_Invalid:
//
// If rSdOffset == SERVICE_TABLE_TEST then service number is GUI service
//
cmp.ne pInvl, pVal = SERVICE_TABLE_TEST, rSdOffset
mov v0 = 1
(pVal) br.call.sptk brp = PsConvertToGuiThread
;;
cmp4.eq pVal, p0 = 0, v0 // pVal = 1, if successful
movl v0 = STATUS_INVALID_SYSTEM_SERVICE // invalid, if not successful
;;
add rpEntry = @gprel(KeServiceDescriptorTableShadow),gp // Load address of table of shadow table
// Used by KiSystemServiceRepeat and the
// fall through case below.
(pVal) br.sptk KiSystemServiceRepeat // br if successful
(pInvl) br.spnt Knsc_ErrorReturn // br to KiSystemServiceExit if
;; // system call out of range
//
// The conversion to a Gui thread failed. The correct return value is encoded
// in a byte table indexed by the service number that is at the end of the
// service address table. The encoding is as follows:
;;
//
// 0 - return 0.
// -1 - return -1.
// 1 - return status code.
//
add rpT1 = SERVICE_TABLE_TEST, rpEntry // Get addr of the GUI table
;;
ld8 rpT2 = [rpT1], SdLimit - SdBase // Get the table base.
;;
ld4 rT1 = [rpT1] // Get the limit.
add rpT2 = rpT2, rSnumber // Index by the server number to the correct byte.
;;
shladd rpT2 = rT1, 3, rpT2 // Calculate the end of the table.
cmp4.ltu pt1, pt0 = rSnumber, rT1 // pt1 = number < limit
;; // pt1 == 0 implies return an ntstatus.
(pt1) ld1 rT1 = [rpT2] // Load the flag byte if within table
;;
sxt1 rT2 = rT1 // Sign extend the result
(pt1) cmp.eq pt0, pt1 = 1, rT1 // Test for 1 which means return STATUS_INVALID_SYSTEM_SERVICE
;;
(pt1) mov v0 = rT2 // Use the flag value if needed.
br.sptk Knsc_ErrorReturn // br to KiSystemServiceExit
;;
//
// Now running with user banked registers and on kernel backing store
//
// Can now take TLB faults
//
// Preserve the output args (as in's) in the local frame until it's ready to
// call the specified system service because other calls may have to be made
// before that. Also allocate locals and one out.
//
Knsc10:
#ifdef _CAPKERN
alloc rUserPFS = 8, 9, 2, 0
#else
alloc rUserPFS = 8, 9, 0, 0
#endif
ld8 rpThObj = [rpT4]
add rpT1 = TrEOFMarker, sp
;;
mov rUserSp = ar.bsp
movl rT1 = FPSR_FOR_KERNEL // initial fpsr value
;;
mov ar.fpsr = rT1 // set fpsr
movl rT3 = KTRAP_FRAME_EOF | SYSCALL_FRAME
;;
st8 [rpT1] = rT3, TrNewBSP - TrEOFMarker
mov rCount = pr // save local predicates
shladd rUserSp = rIFS, 3, rUserSp
;;
st8 [rpT1] = rUserSp // Save NewBSP.
mov rSavedV0 = v0 // save syscall # across call
mov rUserSp = rSp
add rpT2 = ThServiceTable, rpThObj // Address of thread service table
add rpT3 = ThTrapFrame, rpThObj
mov pr = zero, -1
;;
ld8 rT4 = [rpT3], ThPreviousMode-ThTrapFrame
//
// the following code uses the predicates to determine the first of
// the 8 input argument register whose Nat bit is set. The result
// is saved in register rCount and used to determine whether to fail
// the system call in Knsc_CheckArguments.
//
cmp.eq p1 = r32, r32
cmp.eq p9 = r33, r33
cmp.eq p17 = r34, r34
cmp.eq p25 = r35, r35
cmp.eq p33 = r36, r36
cmp.eq p41 = r37, r37
cmp.eq p49 = r38, r38
cmp.eq p57 = r39, r39
ld8 rpEntry = [rpT2] // Load service describtor
add rpT1 = TrTrapFrame, sp // -> TrTrapFrame
shr.u rSdOffset = rSavedV0, SERVICE_TABLE_SHIFT // isolate service descriptor offset
;;
ld1 rT0 = [rpT3] // rT0 = thread's previous mode
st1 [rpT3] = rPreviousMode // set new thread previous mode
mov rT1 = pr
;;
st8 [rpT1] = rT4, TrPreviousMode-TrTrapFrame
dep rT1 = 0, rT1, 0, 1 // clear bit 0
mov pr = rCount, -1 // restore local predicates
;;
and rSdOffset = SERVICE_TABLE_MASK, rSdOffset
extr.u rSnumber = rSavedV0, 0, 12
czx1.r rCount = rT1 // determine which arg is Nat'ed
st4 [rpT1] = rT0
FAST_ENABLE_INTERRUPTS
(pKDbg) br.call.spnt brp = KiLoadKernelDebugRegisters
;;
PROLOGUE_END
//
// If the specified system service number is not within range, then
// attempt to convert the thread to a GUI thread and retry the service
// dispatch.
//
// N.B. The system call arguments, the system service entry point (rpEntry),
// the service number (Snumber)
// are implicitly preserved in the register stack while attempting to
// convert the thread to a GUI thread. v0 and the gp must be preserved
// explicitly.
//
// Validate sys call number
//
#ifdef _CAPKERN
br.call.sptk.few b0=_CAP_ThreadID#;;
#endif
ALTERNATE_ENTRY(KiSystemServiceRepeat)
add rpT1 = ThTrapFrame, rpThObj // rpT1 -> ThTrapFrame
add rpSd = rSdOffset, rpEntry // service descriptor -> rpSd
cmp4.ne pNoGui, pGui = SERVICE_TABLE_TEST, rSdOffset // check if GUI system service
;;
st8 [rpT1] = sp // set trap frame address
ld8 rT4 = [rpSd], SdLimit-SdBase // rT4 = table base
add rpT1 = TeGdiBatchCount, teb
;; // We assume SdBase == 0
#if SdBase
#error "SdBase not equal zero."
#endif
KiSystemServiceTeb::
ld4 rT1 = [rpSd] // rT1 = table limit
(pGui) ld4 rT2 = [rpT1] // get number of batched calls
add rScallGp = -8, rT4 // The GP values is just before the Base
;;
(pGui) cmp4.eq pNoGui, pGui = 0, rT2 // if bacth calls is non-zero then call flush.
cmp4.ltu pt1, pt0 = rSnumber, rT1 // pt1 = number < limit
shladd rpT3 = rSnumber, 3, rT4 // -> entry point address
;;
(pt1) ld8 rScallGp = [rScallGp] // load syscall gp
(pt1) add rpT1 = @gprel(KeGdiFlushUserBatch), gp
(pt0) br.dpnt Knsc_Invalid // The system call is out of range.
;;
ld8 rpEntry = [rpT3] // -> sys call routine plabel
(pGui) ld8 rpT1 = [rpT1] // get KeGdiFlushUserBatch()
(pNoGui) br.dptk Knsc_NotGUI // Not a GUI thread
;;
Knsc_GUI:
//
// If the system service is a GUI service and the GDI user batch queue is
// not empty, then call the appropriate service to flush the user batch.
//
ld8 rT1 = [rpT1], PlGlobalPointer-PlEntryPoint // get entry point
mov rSavedV0 = rpSd // save service descriptor
;;
ld8 gp = [rpT1] // set global pointer
mov bt0 = rT1
br.call.sptk brp = bt0 // call to KeGdiFlushUserBatch
;;
#if DBG
mov rpSd = rSavedV0 // restore service descriptor
;;
#endif
//
// Check for Nat'ed input argument register and
// Copy in-memory arguments from caller stack to kernel stack
//
Knsc_NotGUI:
#ifdef _CAPKERN
movl out0 = @fptr(KiNormalSystemCall)
mov out1 = rpEntry
br.call.sptk.few b0=_CAP_Start_Profiling2
#endif
#if DBG // checked build code
add rpT1 = SdCount-SdLimit, rpSd // rpT1 -> count table address
;;
ld8 rpT2 = [rpT1] // service count table address
;;
cmp.ne pt0 = rpT2, zero // if zero, no table defined
shladd rpT3 = rSnumber, 2, rpT2 // compute service count address
;;
(pt0) ld4 rT1 = [rpT3] // increment count
;;
(pt0) add rT1 = 1, rT1
;;
(pt0) st4 [rpT3] = rT1 // store result
#endif // DBG
and rArgNum = 0xf, rpEntry // extract # of arguments
and rpEntry = -16, rpEntry // clear least significant 4 bit
cmp.ne pNatedArg = zero, zero // assume no Nat'ed argument
mov gp = rScallGp
movl v0 = STATUS_INVALID_PARAMETER_1
;;
cmp.geu pNoCopy, pCopy = 8, rArgNum // any in-memory arguments ?
mov bt0 = rpEntry
mov rSp = rUserSp
;;
add v0 = rCount, v0 // set return status
dep rPFS = 0, rUserPFS, 62, 2
add rArgBytes = -8, rArgNum // rArgBytes contion the number of 16 byte chunks to copy.
;;
(pNoCopy) cmp.gt pNatedArg = rArgNum, rCount // any Nat'ed argument ?
mov ar.pfs = rPFS // Make the user PFS look like our PFS
(pCopy) cmp.gt pNatedArg = 8, rCount
;;
alloc rT1 = 0,0,8,0 // output regs are ready
(pNatedArg) br.spnt Knsc_ErrorReturn // exit if Nat'ed arg found
(pNoCopy) br.sptk KiSystemServiceEnd // skip copy if no memory args
//
// Get the caller's sp. If caller was user mode, validate the stack pointer
// ar.unat contains Nat for previous sp (in TrIntSp)
//
(pUser) movl rT2 = MI_USER_PROBE_ADDRESS // User sp limit
;;
add rArgBytes = -1, rArgBytes // decrement count
(pUser) tnat.nz.unc pt1, pt2 = rSp // test user sp for Nat
add rpT1 = STACK_SCRATCH_AREA, rSp // adjust previous sp for scratch area
;;
(pt2) cmp.geu pt1 = rSp, rT2 // user sp >= PROBE ADDRESS ?
add rpT2 = STACK_SCRATCH_AREA, sp // adjust for scratch area
;;
add rpT3 = 8, rpT1 // second source pointer
add rpT4 = 8, rpT2 // second destination pointer
(pt1) mov rpT1 = rT2 // set out of range (includes Nat case)
;;
//
// At this point rpT1, rpT3 -> source and rpT2, rpT4 -> destination
// Copy rArgBytes from source to destination, 16 bytes per iteration
// Exceptions handled by KiSystemServiceHandler
//
KiSystemServiceStart::
Knsc_CopyLoop:
ld8 rT1 = [rpT1], 16 // get caller arg
ld8 rT2 = [rpT3], 16 // get caller arg
cmp4.gt pt1 = rArgBytes, zero // loop if # bytes > 0
;;
nop.m 0
nop.f 0
add rArgBytes = -1, rArgBytes // decrement count
st8 [rpT2] = rT1, 16 // store in kernel stack
st8 [rpT4] = rT2, 16 // store in kernel stack
(pt1) br.dpnt Knsc_CopyLoop
;;
KiSystemServiceEnd::
//
// N.B. t0 is reserved to pass trap frame address to NtContinue()
//
mov t0 = sp // for NtContinue()
movl rT1 = KiSystemServiceExit
;;
//
// This is not call since that would change the stack registers.
// we want to got to the routine with the same argument regisers
// that we entered with.
//
mov brp = rT1
br.sptk bt0 // call routine(args)
;;
KiSystemServiceExit::
//
// make the current bsp the same as the saved BSP in the trap frame
//
add rpT2 = TrStIPSR, sp
cover
;;
#ifdef _CAPKERN
alloc rT1 = ar.pfs, 0, 0, 1, 0
movl gp = _gp // restore to kernel GP value
;;
CAPEND(KiNormalSystemCall)
br.call.sptk.few b0=_CAP_SetCPU#
;;
add rpT2 = TrStIPSR, sp
;;
#endif
//
// At this point:
// ar.unat contains Nat for previous sp (ar.unat is preserved register)
// sp -> trap frame
//
// Returning from "call": no need to restore volatile state
// *** TBD *** : zero volatile state for security? PPC does zero, mips does not.
//
//
// Update PbSystemCalls
//
//
// Restore thread previous mode and trap frame address from the trap frame
//
lfetch [rpT2], TrTrapFrame-TrStIPSR
movl rpT1 = KiPcr + PcPrcb // rpT1 -> Prcb
;;
ld8 rT2 = [rpT2], TrPreviousMode-TrTrapFrame
ld8 rpT3 = [rpT1], PcCurrentThread-PcPrcb
;;
FAST_DISABLE_INTERRUPTS
ld8 rThread = [rpT1] // rpT1 -> current thread
add rpT3 = PbSystemCalls, rpT3 // pointer to sys call counter
;;
ld4 rT1 = [rpT3] // rT1.4 = counter value
ld4 rT3 = [rpT2], TrRsRSC-TrPreviousMode
add rpT4 = ThTrapFrame, rThread // -> thread trap frame
add rpT1 = ThApcState+AsUserApcPending, rThread
;;
(pKrnl) ld8 rRscE = [rpT2] // load user RSC
(pUser) ld1 rT4 = [rpT1], ThAlerted - ThApcState - AsUserApcPending
add rT1 = 1, rT1 // increment
;;
st8 [rpT4] = rT2, ThPreviousMode-ThTrapFrame
st4 [rpT3] = rT1 // store
;;
st1 [rpT4] = rT3 // restore prevmode in thread
mov t0 = sp // set t0 to trap frame
(pKrnl) br.spnt Knsc_CommonExit // br if returning to kernel
;;
st1 [rpT1] = zero
cmp4.eq pt0 = zero, rT4
(pt0) br.sptk KiUserServiceExit
;;
alloc rT1 = ar.pfs, 0, 1, 2, 0
add loc0 = TrIntV0, sp
add rpT1 = TrIntTeb, sp
//
// v0 is saved in the trap frame so the return status can be restored
// by NtContinue after the user APC has been dispatched.
//
ssm 1 << PSR_I // enable interrupts
mov rT3 = APC_LEVEL
;;
SET_IRQL (rT3)
;;
st8.nta [loc0] = v0 // save return status in trap frame
movl gp = _gp // restore to kernel gp value
st8.nta [rpT1] = teb
#ifdef _CAPKERN
mov rpT1 = out0
;;
CAPSTART(KiNormalSystemCall,KiApcInterrupt)
mov out0 = rpT1
#endif
mov out1 = sp
br.call.sptk brp = KiApcInterrupt
;;
CAPEND(KiNormalSystemCall)
rsm 1 << PSR_I // disable interrupt
ld8.nta v0 = [loc0] // restore system call return status
SET_IRQL (zero)
HANDLER_EXIT(KiNormalSystemCall)
//
// KiServiceExit is carefully constructed as a continuation of
// KiNormalSystemCall. From now on, t0 must be preserved because it is
// used to hold the trap frame address. v0 must be preserved because it
// is holding the return status.
// This section of code is also written to overwrite the tempoary interger
// registers so that no information is leaked back to the user.
//
HANDLER_ENTRY_EX(KiServiceExit, KiSystemServiceHandler)
.prologue
.unwabi @nt, EXCEPTION_FRAME
.vframe t0
mov t0 = sp
;;
ALTERNATE_ENTRY(KiUserServiceExit)
(pUser) add rpT3 = TrStIPSR, t0
;;
(pUser) ld8 rpT3 = [rpT3]
invala
;;
(pUser) tbit.nz.unc pUDbg = rpT3, PSR_DB // if user psr.db set, load user DRs
(pUDbg) br.call.spnt brp = KiLoadUserDebugRegisters
;;
add rpT1 = TrRsRSC, t0 // -> user RSC
add rpT2 = TrRsBSP, t0 // -> user BSP Store
;;
PROLOGUE_END
ld8 rRscE = [rpT1], TrRsRNAT-TrRsRSC // load user RSC
ld8 rBSP = [rpT2]
mov rRscD = RSC_KERNEL_DISABLED
;;
//
// Switch to user BSP -- put in load intensive mode to overlap RS restore
// with volatile state restore.
//
ld8 rRNAT = [rpT1] // user RNAT
extr.u rRscDelta = rRscE, RSC_MBZ1, RSC_LOADRS_LEN
dep rRscE = r0, rRscE, RSC_MBZ1, RSC_LOADRS_LEN
movl rpT1 = KiIA64RseNumOfMaxDirty
;;
.regstk 0, 0, 1, 0
alloc rT2 = 0,0,1,0
ld8 rpT1 = [rpT1]
dep rRscD = rRscDelta, rRscD, RSC_LOADRS, RSC_LOADRS_LEN
sub rBSPStore = rBSP, rRscDelta
shr.u out0 = rRscDelta, 3
;;
cmp.gt pt0 = rpT1, out0
sub out0 = rpT1, out0
(pt0) br.call.sptk brp = KeScrubBackingStoreRegisters
;;
alloc rT2 = 0, 0, 0, 0
mov ar.rsc = rRscD // turn off RSE
;;
loadrs // pull in user regs
;;
mov ar.bspstore = rBSPStore // restore user BSP
;;
mov ar.rnat = rRNAT // restore user RNAT
Knsc_CommonExit:
add rpT1 = TrIntSp, t0
movl rpT2 = KiPcr+PcCurrentThread
add rpT4 = TrStIPSR, t0
add rpT3 = TrStIIP, t0
;;
ld8 rSp = [rpT1], TrPreds-TrIntSp
.pred.rel "mutex",pUser,pKrnl
(pUser) ld8 rT1 = [rpT2], PcHighFpOwner-PcCurrentThread
(pKrnl) add rpT2 = PcKernelDebugActive-PcCurrentThread, rpT2
;;
ld8 rIPSR = [rpT4], TrStIFS-TrStIPSR
(pUser) ld8 rT2 = [rpT2]
(pKrnl) mov rPreviousMode = KernelMode
;;
ld8 rIIP = [rpT3], TrStFPSR-TrStIIP
(pKrnl) ld1 rT2 = [rpT2]
(pUser) mov rPreviousMode = UserMode
;;
mov ar.rsc = rRscE // restore RSC
(pUser) cmp.ne.unc pt0 = rT1, rT2
;;
ld8 rT3 = [rpT3] // load fpsr
ld8 rIFS = [rpT4] // load IFS
(pt0) dep rIPSR = 1, rIPSR, PSR_DFH, 1
;;
(pKrnl) cmp.ne.unc pKDbg, pt2 = rT2, r0 // hardware debug active?
dep rIPSR = 0, rIPSR, PSR_MFH, 1 // psr.mfh
;;
ld8 rPreds = [rpT1], TrApUNAT-TrPreds
(pt2) dep rIPSR = 0, rIPSR, PSR_DB, 1 // disable db in kernel
(pKDbg) dep rIPSR = 1, rIPSR, PSR_DB, 1 // enable db in kernel
;;
ld8 rUNAT = [rpT1]
mov t11 = r0 // Sanitize
mov pr = rPreds // restore preds
;;
mov ar.csd = r0 // Sanitize CSD
rsm 1 << PSR_IC // disable PSR.ic
mov ar.unat = rUNAT // restore UNAT
;;
srlz.d // must serialize
mov ar.fpsr = rT3 // restore FPSR
;;
//
// The system return case must be handle separately from the user return
// case, so we can determine which stack we are currently using in case
// a trap is take before we can return.
//
mov sp = rSp
//
// Restore status registers
//
mov cr.ipsr = rIPSR // restore previous IPSR
mov cr.ifs = rIFS // restore previous IFS
mov cr.iip = rIIP // restore previous IIP
;;
//
// Resume at point of interruption (rfi must be at end of instruction group)
//
rfi
;;
Knsc_ErrorReturn:
//
// N.B. t0 is reserved to pass trap frame address to NtContinue()
//
add rPFS = TrRsPFS, sp
;;
ld8 rPFS = [rPFS]
;;
dep rPFS = 0, rPFS, 62, 2
;;
mov ar.pfs = rPFS
movl rT1 = KiSystemServiceExit
;;
mov brp = rT1
br.ret.sptk brp
;;
HANDLER_EXIT(KiServiceExit)
.text
//++
//--------------------------------------------------------------------
// Routine:
//
// KiGenericExceptionHandler
//
// Description:
//
// First level handler for heavyweight exceptions.
//
// On entry:
//
// ic off
// interrupts disabled
// current frame covered
// rHIFA must be initialized to a non-nated value
//
// Process:
//
// Notes:
//
// PCR page mapped with TR
//--------------------------------------------------------------------
HANDLER_ENTRY(KiGenericExceptionHandler)
.prologue
.unwabi @nt, EXCEPTION_FRAME
ALLOCATE_TRAP_FRAME
//
// sp points to trap frame
//
// Save exception handler routine in kernel register
//
mov rkHandler = rHHandler
movl rTH3 = KiPcr+PcSavedIFA
;;
st8 [rTH3] = rHIFA
//
// Save interruption state in trap frame and switch to user bank registers
// and switch to kernel backing store.
//
SAVE_INTERRUPTION_STATE
//
// Now running with user banked registers and on kernel stack.
//
// Can now take TLB faults
//
// sp -> trap frame
//
br.call.sptk brp = KiSaveTrapFrame
;;
//
// Register aliases
//
rpT1 = t0
rpT2 = t1
rpT3 = t2
rT1 = t3
rT2 = t4
rT3 = t5
rPreviousMode = t6 // previous mode
rT4 = t7
mov rT1 = rkHandler // restore address of interruption routine
movl rpT1 = KiPcr+PcSavedIIM
;;
ld8 rT2 = [rpT1], PcSavedIFA-PcSavedIIM // load saved IIM
add rpT2 = TrEOFMarker, sp
add rpT3 = TrStIIM, sp
;;
ld8 rT4 = [rpT1] // load saved IFA
movl rT3 = KTRAP_FRAME_EOF | EXCEPTION_FRAME
;;
mov t8 = ar.bsp
st8 [rpT2] = rT3, TrNewBSP-TrEOFMarker
st8 [rpT3] = rT2, TrStIFA-TrStIIM // save IIM in trap frame
;;
.regstk 0, 1, 2, 0 // must match KiExceptionExit
alloc out1 = 0,1,2,0
st8 [rpT3] = rT4 // save IFA in trap frame
mov bt0 = rT1 // set destination address
PROLOGUE_END
//
// Dispatch the exception via call to address in rkHandler
//
.pred.rel "mutex",pUser,pKrnl
add rpT1 = TrPreviousMode, sp // -> previous mode
(pUser) mov rPreviousMode = UserMode // set previous mode
(pKrnl) mov rPreviousMode = KernelMode
;;
st4 [rpT1] = rPreviousMode // save in trap frame
st8 [rpT2] = t8 // Save new BSP for debugging
(pKDbg) br.call.spnt brp = KiLoadKernelDebugRegisters
;;
FAST_ENABLE_INTERRUPTS // enable interrupt
mov out0 = sp // trap frame pointer
br.call.sptk brp = bt0 // call handler(tf) (C code)
;;
.pred.rel "mutex",pUser,pKrnl
cmp.ne pt0, pt1 = v0, zero
(pUser) mov out1 = UserMode
(pKrnl) mov out1 = KernelMode
//
// does not return
//
mov out0 = sp
(pt1) br.cond.sptk KiAlternateExit
(pt0) br.call.spnt brp = KiExceptionDispatch
;;
//
// Interrupts need to be disable be for mess up the stack such that
// the unwind code does not work.
//
FAST_DISABLE_INTERRUPTS
;;
ALTERNATE_ENTRY(KiExceptionExit)
//++
//
// Routine Description:
//
// This routine is called to exit from an exception.
//
// N.B. This transfer of control occurs from:
//
// 1. fall-through from above
// 2. exit from continue system service
// 3. exit from raise exception system service
// 4. exit into user mode from thread startup
//
// Arguments:
//
// loc0 - pointer to trap frame
// sp - pointer to high preserved float save area + STACK_SCRATCH_AREA
//
// Return Value:
//
// Does not return.
//
//--
//
// upon entry of this block, s0 and s1 must be set to the address of
// the trap and the exception frames respectively.
//
// preserved state is restored here because they may have been modified
// by SetContext
//
LEAF_SETUP(0, 1, 2, 0) // must be in sync with
// KiGenericExceptionHandler
mov loc0 = s0 // -> trap frame
mov out0 = s1 // -> exception frame
;;
br.call.sptk brp = KiRestoreExceptionFrame
;;
mov sp = loc0 // deallocate exception
// frame by restoring sp
ALTERNATE_ENTRY(KiAlternateExit)
//
// sp -> trap frame addres
//
// Interrupts disabled from here to rfi
//
FAST_DISABLE_INTERRUPTS
;;
RETURN_FROM_INTERRUPTION(Ked)
//
// Add four nop cycles (two bundles) bundles of NOP's to
// workaround McKinly Errata.
//
nop.m 0
;;
nop.m 0
;;
nop.m 0
;;
nop.m 0
;;
HANDLER_EXIT(KiGenericExceptionHandler)
//--------------------------------------------------------------------
// Routine:
//
// KiExternalInterruptHandler
//
// Description:
//
// First level external interrupt handler. Dispatch highest priority
// pending interrupt.
//
// On entry:
//
// ic off
// interrupts disabled
// current frame covered
//
// Process:
//--------------------------------------------------------------------
HANDLER_ENTRY(KiExternalInterruptHandler)
//
// Now running with user banked registers and on kernel backing store.
// N.B. sp -> trap frame
//
// Can now take TLB faults
//
.prologue
.unwabi @nt, INTERRUPT_FRAME
.regstk 0, 4, 2, 0
alloc loc0 = 0, 4, 2, 0
//
// Register aliases
//
rVector = loc0
rSaveGP = loc1
rpSaveIrql = loc2 // -> old irql in trap frame
rOldIrql = loc3
rpT1 = t0
rpT2 = t1
rpT3 = t2
rT1 = t3
rT2 = t4
rT3 = t5
rPreviousMode = t6 // previous mode
rNewIrql = t7
pEOI = pt1
movl rOldIrql = KiPcr+PcCurrentIrql
;;
//
// Save kernel gp
//
mov rSaveGP = gp
//
// Get the vector number
//
mov rVector = cr.ivr // for A0 2173 workaround
br.call.sptk brp = KiSaveTrapFrame
;;
#if defined(INTERRUPTION_LOGGING)
movl t0 = KiVectorLogMask
;;
ld8 t1 = [t0]
;;
tbit.z pt1 = t1, ExternalInterruptBit
(pt1) br.cond.sptk EndOfLogging2
movl t0 = KiPcr+PcInterruptionCount
;;
ld4.nt1 t0 = [t0]
mov t1 = MAX_NUMBER_OF_IHISTORY_RECORDS - 1
;;
add t0 = -1, t0
movl t2 = KiPcr+0x1000
;;
and t1 = t0, t1
;;
shl t1 = t1, 5
;;
add t0 = t2, t1
;;
add t0 = 24, t0
;;
st8.nta [t0] = rVector // save ivr in the Extra0 field
// For Conditional Interrupt Logging
EndOfLogging2:
#endif // defined(INTERRUPTION_LOGGING)
//
// Exit if spurious interrupt vector
//
cmp.eq pt0, pt1 = 0xF, rVector
ld1 rOldIrql = [rOldIrql] // The Irql in the PCR has been santized in
// SAVE_INTERRUPTION_STATE.
(pt0) br.spnt Keih_Exit
;;
//
// sp -> trap frame
//
add rpSaveIrql = TrEOFMarker, sp
movl rT3 = KTRAP_FRAME_EOF | INTERRUPT_FRAME
;;
st8 [rpSaveIrql] = rT3, TrPreviousMode - TrEOFMarker
.pred.rel "mutex",pUser,pKrnl
(pUser) mov rPreviousMode = UserMode // set previous mode
(pKrnl) mov rPreviousMode = KernelMode
;;
st4 [rpSaveIrql] = rPreviousMode, TrOldIrql-TrPreviousMode
;;
st4 [rpSaveIrql] = rOldIrql // save irql in trap frame
(pKDbg) br.call.spnt brp = KiLoadKernelDebugRegisters
;;
PROLOGUE_END
Keih_InterruptLoop:
//
// Dispatch the interrupt: first raise the IRQL to the level of the new
// interrupt and enable interrupts.
//
GET_IRQL_FOR_VECTOR(p0, rNewIrql, rVector)
movl rpT3 = KiPcr+PcInterruptRoutine // -> interrupt routine table
;;
shladd rpT3 = rVector, INT_ROUTINES_SHIFT, rpT3 // base + offset
SET_IRQL (rNewIrql) // raise to new level
movl rpT1 = KiPcr + PcPrcb // pointer to prcb
;;
ld8 out0 = [rpT3] // out0 -> interrupt dispatcher
ld8 rpT1 = [rpT1]
mov out1 = sp // out1 -> trap frame
;;
FAST_ENABLE_INTERRUPTS
add rpT1 = PbInterruptCount, rpT1 // -> interrupt counter
;;
ld4.nta rT1 = [rpT1] // counter value
ld8.nta rT2 = [out0], PlGlobalPointer-PlEntryPoint // get entry point
;;
add rT1 = 1, rT1 // increment
;;
//
// Call the interrupt dispatch routine via a function pointer
//
st4.nta [rpT1] = rT1 // store, ignore overflow
;;
ld8.nta gp = [out0], PlEntryPoint-PlGlobalPointer
mov bt0 = rT2
br.call.sptk brp = bt0 // call ISR
;;
ld4 rOldIrql = [rpSaveIrql]
mov gp = rSaveGP
END_OF_INTERRUPT // end of interrupt processing
IO_END_OF_INTERRUPT(rVector,rT1,rT2,pEOI)
;;
srlz.d
//
// Disable interrupts and restore IRQL level
//
FAST_DISABLE_INTERRUPTS
cmp.gt pt0 = APC_LEVEL, rOldIrql
mov t0 = APC_LEVEL
;;
(pt0) br.spnt ke_call_apc
ke_call_apc_resume:
LOWER_IRQL (rOldIrql)
//
// Get the next vector number
//
mov rVector = cr.ivr // for A0 2173 workaround
;;
//
// Loop if more interrupts pending (spurious vector == 0xF)
//
cmp.ne pt0 = 0xF, rVector
;;
(pt0) br.spnt Keih_InterruptLoop
br.sptk Keih_Exit
;;
ke_call_apc:
LOWER_IRQL (t0)
ke_call_apc1:
movl t0 = KiPcr+PcApcInterrupt
;;
ld1 t1 = [t0]
st1 [t0] = r0
;;
cmp.eq pt0 = r0, t1
;;
(pt0) br.sptk ke_call_apc_resume
FAST_ENABLE_INTERRUPTS
mov out1 = sp
br.call.sptk brp = KiApcInterrupt
FAST_DISABLE_INTERRUPTS
br.sptk ke_call_apc1
Keih_Exit:
RETURN_FROM_INTERRUPTION(Keih)
HANDLER_EXIT(KiExternalInterruptHandler)
//--------------------------------------------------------------------
// Routine:
//
// KiPanicHandler
//
// Description:
//
// Handler for panic. Call the bug check routine. A place
// holder for now.
//
// On entry:
//
// running on kernel memory stack and kernel backing store
// sp: top of stack -- points to trap frame
// interrupts enabled
//
// IIP: address of bundle causing fault
//
// IPSR: copy of PSR at time of interruption
//
// Output:
//
// sp: top of stack -- points to trap frame
//
// Return value:
//
// none
//
// Notes:
//
// If ISR code out of bounds, this code will inovke the panic handler
//
//--------------------------------------------------------------------
HANDLER_ENTRY(KiPanicHandler)
.prologue
.unwabi @nt, EXCEPTION_FRAME
mov ar.k1 = rTH1 // Save Old Stack Limit
mov ar.k2 = rTH2 // Save Old BSP Limit
mov ar.k4 = rHVector
mov rkHandler = rHHandler
;;
mov rHpT1 = KERNEL_STACK_SIZE
movl rTH1 = KiPcr+PcPanicStack
;;
ld8 sp = [rTH1], PcInitialStack-PcPanicStack
movl rTH2 = KiPcr+PcSystemReserved
;;
st4 [rTH2] = rPanicCode
st8 [rTH1] = sp, PcStackLimit-PcInitialStack
sub rTH2 = sp, rHpT1
;;
st8 [rTH1] = rTH2, PcInitialBStore-PcStackLimit
mov rHpT1 = KERNEL_BSTORE_SIZE
;;
st8 [rTH1] = sp, PcBStoreLimit-PcInitialBStore
add rTH2 = rHpT1, sp
add sp = -TrapFrameLength, sp
;;
st8 [rTH1] = rTH2, PcSavedIFA - PcBStoreLimit
;;
st8 [rTH1] = rHIFA, PcSavedIIM - PcSavedIFA
mov rTH2 = cr.iha
;;
st8 [rTH1] = rTH2 // save iha in the iim field
SAVE_INTERRUPTION_STATE
//
// switch to kernel back
//
bsw.0
;;
rpRNAT = h16
rpBSPStore= h17
rBSPStore = h18
rKBSPStore= h19
rRNAT = h20
rKrnlFPSR = h21
mov ar.rsc = r0 // put RSE in lazy mode
movl rKBSPStore = KiPcr+PcInitialBStore
;;
mov rBSPStore = ar.bspstore
mov rRNAT = ar.rnat
;;
ld8 rKBSPStore = [rKBSPStore]
add rpRNAT = TrRsRNAT, sp
add rpBSPStore = TrRsBSPSTORE, sp
;;
st8 [rpRNAT] = rRNAT
st8 [rpBSPStore] = rBSPStore
dep rKBSPStore = rBSPStore, rKBSPStore, 0, 9
;;
mov ar.bspstore = rKBSPStore
mov ar.rsc = RSC_KERNEL
;;
//
// switch to user bank
//
bsw.1
;;
alloc out0 = ar.pfs, 0, 0, 5, 0
;;
PROLOGUE_END
br.call.sptk brp = KiSaveTrapFrame
;;
movl rpT1 = KiPcr+PcSavedIIM
;;
ld8 rT2 = [rpT1], PcSavedIFA-PcSavedIIM // load iha
;;
ld8 rT3 = [rpT1] // load ifa
add rpT3 = TrStIHA, sp
;;
st8 [rpT3] = rT2, TrStIFA-TrStIHA
;;
st8 [rpT3] = rT3
// Raise the IRQL to HIGH
add rpT2 = TrOldIrql, sp
movl rpT1 = KiPcr + PcCurrentIrql
;;
ld1 rT1 = [rpT1]
mov rT2 = HIGH_LEVEL
;;
st1 [rpT2] = rT1
st1 [rpT1] = rT2
dep.z rT3 = rT2, TPR_MIC, TPR_MIC_LEN
;;
mov cr.tpr = rT3
mov out2 = ar.k4 // 3rd argument: caller of panic
mov out3 = rkHandler
movl out0 = KiPcr+PcSystemReserved
;;
ld4 out0 = [out0] // 1st argument: panic code
mov out1 = sp // 2nd argument: trap frame
br.call.sptk.many brp = KeBugCheckEx
;;
nop.m 0
nop.m 0
nop.i 0
;;
HANDLER_EXIT(KiPanicHandler)
//++
//--------------------------------------------------------------------
// Routine:
//
// VOID
// KiSaveTrapFrame(PKTRAP_FRAME)
//
// Description:
//
// Save volatile application state in trap frame.
// Note: sp, brp, UNAT, RSC, predicates, BSP, BSP Store,
// PFS, CSD, and FPSR saved elsewhere.
//
// Input:
//
// sp: points to trap frame
// ar.unat: contains the Nats of sp, gp, teb, which have already
// been spilled into the trap frame.
//
// Output:
//
// None
//
// Return value:
//
// none
//
//--------------------------------------------------------------------
LEAF_ENTRY(KiSaveTrapFrame)
.regstk 0, 3, 0, 0
//
// Local register aliases
//
rpTF1 = loc0
rpTF2 = loc1
rL1 = t0
rL2 = t1
rL3 = t2
rL4 = t3
rL5 = t4
//
// (ar.unat unchanged from point of save)
// Spill temporary (volatile) integer registers
//
alloc loc2 = 0,3,0,0 // don't destroy static register
add rpTF1 = TrIntT0, sp // -> t0 save area
add rpTF2 = TrIntT1, sp // -> t1 save area
;;
.mem.offset 0,0
st8.spill [rpTF1] = t0, TrIntT2-TrIntT0 // spill t0 - t22
.mem.offset 8,0
st8.spill [rpTF2] = t1, TrIntT3-TrIntT1
;;
.mem.offset 0,0
st8.spill [rpTF1] = t2, TrIntT4-TrIntT2
.mem.offset 8,0
st8.spill [rpTF2] = t3, TrIntT5-TrIntT3
;;
.mem.offset 0,0
st8.spill [rpTF1] = t4, TrIntT6-TrIntT4
.mem.offset 8,0
st8.spill [rpTF2] = t5, TrIntT7-TrIntT5
mov rL2 = bt0
;;
mov rL1 = ar.csd
mov rL4 = ar.ccv
mov rL3 = bt1
;;
.mem.offset 0,0
st8.spill [rpTF1] = t6, TrIntT8-TrIntT6
.mem.offset 8,0
st8.spill [rpTF2] = t7, TrIntT9-TrIntT7
mov t5 = cr.tpr // Capture the real IRQL
;;
.mem.offset 0,0
st8.spill [rpTF1] = t8, TrIntT10-TrIntT8
.mem.offset 8,0
st8.spill [rpTF2] = t9, TrIntT11-TrIntT9
;;
.mem.offset 0,0
st8.spill [rpTF1] = t10, TrIntT12-TrIntT10
.mem.offset 8,0
st8.spill [rpTF2] = t11, TrIntT13-TrIntT11
;;
.mem.offset 0,0
st8.spill [rpTF1] = t12, TrIntT14-TrIntT12
.mem.offset 8,0
st8.spill [rpTF2] = t13, TrIntT15-TrIntT13
;;
.mem.offset 0,0
st8.spill [rpTF1] = t14, TrIntT16-TrIntT14
.mem.offset 8,0
st8.spill [rpTF2] = t15, TrIntT17-TrIntT15
;;
.mem.offset 0,0
st8.spill [rpTF1] = t16, TrIntT18-TrIntT16
.mem.offset 8,0
st8.spill [rpTF2] = t17, TrIntT19-TrIntT17
;;
.mem.offset 0,0
st8.spill [rpTF1] = t18, TrIntT20-TrIntT18
.mem.offset 8,0
st8.spill [rpTF2] = t19, TrIntT21-TrIntT19
;;
.mem.offset 0,0
st8.spill [rpTF1] = t20, TrIntT22-TrIntT20
.mem.offset 8,0
st8.spill [rpTF2] = t21, TrIntV0-TrIntT21
;;
.mem.offset 0,0
st8.spill [rpTF1] = t22, TrBrT0-TrIntT22
.mem.offset 8,0
st8.spill [rpTF2] = v0, TrBrT1-TrIntV0 // spill old V0
;;
st8 [rpTF1] = rL2, TrApCCV-TrBrT0 // save old bt0 - bt1
st8 [rpTF2] = rL3
mov rL5 = ar.unat
;;
st8 [rpTF1] = rL4, TrSegCSD-TrApCCV // save ar.ccv
movl rL3 = KiPcr+PcCurrentIrql
;;
st8 [rpTF1] = rL1 // save ar.csd
add rpTF1 = TrFltT0, sp // point to FltT0
add rpTF2 = TrFltT1, sp // point to FltT1
;;
//
// Spill temporary (volatile) floating point registers
//
stf.spill [rpTF1] = ft0, TrFltT2-TrFltT0 // spill float tmp 0 - 9
stf.spill [rpTF2] = ft1, TrFltT3-TrFltT1
;;
stf.spill [rpTF1] = ft2, TrFltT4-TrFltT2
stf.spill [rpTF2] = ft3, TrFltT5-TrFltT3
;;
stf.spill [rpTF1] = ft4, TrFltT6-TrFltT4
stf.spill [rpTF2] = ft5, TrFltT7-TrFltT5
;;
stf.spill [rpTF1] = ft6, TrFltT8-TrFltT6
stf.spill [rpTF2] = ft7, TrFltT9-TrFltT7
add t20 = TrIntNats, sp
;;
stf.spill [rpTF1] = ft8
stf.spill [rpTF2] = ft9
extr.u t5 = t5, TPR_MIC, TPR_MIC_LEN
;;
st8 [t20] = rL5 // save volatile iNats
st1 [rL3] = t5 // Sanitize currnet IRQL in PCR.
LEAF_RETURN
;;
LEAF_EXIT(KiSaveTrapFrame)
//++
//--------------------------------------------------------------------
// Routine:
//
// VOID
// KiRestoreTrapFrame(PKTRAP_FRAME)
//
// Description:
//
// Restore volatile application state from trap frame. Restore CSD
// Note: sp, brp, RSC, UNAT, predicates, BSP, BSP Store, PFS,
// CSD and FPSR not restored here.
//
// Input:
//
// sp: points to trap frame
// RSE frame size is zero
//
// Output:
//
// None
//
// Return value:
//
// none
//
//--------------------------------------------------------------------
LEAF_ENTRY(KiRestoreTrapFrame)
LEAF_SETUP(0,2,0,0)
rpTF1 = loc0
rpTF2 = loc1
mov t21 = psr
add t12 = TrIntNats, sp
add rpTF2 = TrApCCV, sp
;;
ld8 t0 = [t12], TrBrT0-TrIntNats
ld8 t1 = [rpTF2], TrBrT1-TrApCCV
add rpTF1 = TrFltT0, sp
;;
ld8 t2 = [t12], TrStFPSR-TrBrT0
ld8 t3 = [rpTF2], TrSegCSD-TrBrT1
;;
ld8 t10 = [t12]
ld8 t11 = [rpTF2], TrFltT1-TrSegCSD
;;
ldf.fill ft0 = [rpTF1], TrFltT2-TrFltT0
ldf.fill ft1 = [rpTF2], TrFltT3-TrFltT1
;;
mov ar.unat = t0
ldf.fill ft2 = [rpTF1], TrFltT4-TrFltT2
ldf.fill ft3 = [rpTF2], TrFltT5-TrFltT3
;;
ldf.fill ft4 = [rpTF1], TrFltT6-TrFltT4
ldf.fill ft5 = [rpTF2], TrFltT7-TrFltT5
;;
ldf.fill ft6 = [rpTF1], TrFltT8-TrFltT6
ldf.fill ft7 = [rpTF2], TrFltT9-TrFltT7
;;
ldf.fill ft8 = [rpTF1], TrIntGp-TrFltT8
ldf.fill ft9 = [rpTF2], TrIntT0-TrFltT9
;;
mov ar.ccv = t1
ld8.fill gp = [rpTF1], TrIntT1-TrIntGp
;;
ld8.fill t0 = [rpTF2], TrIntT2-TrIntT0
ld8.fill t1 = [rpTF1], TrIntT3-TrIntT1
mov bt0 = t2
;;
mov ar.csd = t11
ld8.fill t2 = [rpTF2], TrIntT4-TrIntT2
mov bt1 = t3
;;
ld8.fill t3 = [rpTF1], TrIntT5-TrIntT3
tbit.z pt1 = t21, PSR_MFL
mov ar.fpsr = t10
ld8.fill t4 = [rpTF2], TrIntT6-TrIntT4
;;
ld8.fill t5 = [rpTF1], TrIntT7-TrIntT5
ld8.fill t6 = [rpTF2], TrIntT8-TrIntT6
;;
ld8.fill t7 = [rpTF1], TrIntT9-TrIntT7
ld8.fill t8 = [rpTF2], TrIntT10-TrIntT8
;;
ld8.fill t9 = [rpTF1], TrIntT11-TrIntT9
ld8.fill t10 = [rpTF2], TrIntT12-TrIntT10
;;
ld8.fill t11 = [rpTF1], TrIntT13-TrIntT11
ld8.fill t12 = [rpTF2], TrIntT14-TrIntT12
;;
ld8.fill t13 = [rpTF1], TrIntT15-TrIntT13
ld8.fill t14 = [rpTF2], TrIntT16-TrIntT14
;;
ld8.fill t15 = [rpTF1], TrIntT17-TrIntT15
ld8.fill t16 = [rpTF2], TrIntT18-TrIntT16
;;
ld8.fill t17 = [rpTF1], TrIntT19-TrIntT17
ld8.fill t18 = [rpTF2], TrIntT20-TrIntT18
;;
ld8.fill t19 = [rpTF1], TrIntT21-TrIntT19
ld8.fill t20 = [rpTF2], TrIntT22-TrIntT20
;;
ld8.fill t21 = [rpTF1], TrIntTeb-TrIntT21
ld8.fill t22 = [rpTF2], TrIntV0-TrIntT22
;;
ld8.fill teb = [rpTF1]
ld8.fill v0 = [rpTF2]
br.ret.sptk.many brp
;;
LEAF_EXIT(KiRestoreTrapFrame)
//++
//--------------------------------------------------------------------
// Routine:
//
// VOID
// KiLoadKernelDebugRegisters
//
// Description:
//
// We maintain two debug register flags:
// 1. Thread DebugActive: Debug registers active for current thread
// 2. PCR KernelDebugActive: Debug registers active in kernel mode
// (setup by kernel debugger)
//
// On user -> kernel transitions there are four possibilities:
//
// Thread Kernel
// DebugActive DebugActive Action
//
// 1. 0 0 None
//
// 2. 1 0 None (kernel PSR.db = 0 by default)
//
// 3. 0 1 Set PSR.db = 1 for kernel
//
// 4. 1 1 Set PSR.db = 1 for kernel and
// load kernel debug registers
//
// Note we never save the user debug registers:
// the user cannot change the DRs so the values in the DR save area are
// always up-to-date (set by SetContext).
//
// Input:
//
// None (Previous mode is USER)
//
// Output:
//
// None
//
// Return value:
//
// none
//
//--------------------------------------------------------------------
NESTED_ENTRY(KiLoadKernelDebugRegisters)
PROLOGUE_BEGIN
movl t10 = KiPcr+PcPrcb
;;
ld8 t10 = [t10] // load prcb address
cmp.eq pt3, pt2 = r0, r0
;;
add t11 = PbProcessorState+KpsSpecialRegisters+KsKernelDbI0,t10
add t12 = PbProcessorState+KpsSpecialRegisters+KsKernelDbI1,t10
add t13 = PbProcessorState+KpsSpecialRegisters+KsKernelDbD0,t10
add t14 = PbProcessorState+KpsSpecialRegisters+KsKernelDbD1,t10
br Krdr_Common
;;
ALTERNATE_ENTRY(KiLoadUserDebugRegisters)
//
// Restore debug registers, if debug active
//
cmp.ne pt3, pt2 = r0, r0
movl t10 = KiPcr+PcCurrentThread
;;
ld8 t10 = [t10] // get current thread pointer
;;
add t10 = ThStackBase, t10
;;
ld8.nta t10 = [t10] // get stack base
;;
add t11 = -ThreadStateSaveAreaLength+TsDebugRegisters+DrDbI0,t10
add t12 = -ThreadStateSaveAreaLength+TsDebugRegisters+DrDbI1,t10
add t13 = -ThreadStateSaveAreaLength+TsDebugRegisters+DrDbD0,t10
add t14 = -ThreadStateSaveAreaLength+TsDebugRegisters+DrDbD1,t10
;;
Krdr_Common:
.regstk 0, 2, 2, 0
.save ar.pfs, savedpfs
alloc savedpfs = ar.pfs, 0, 2, 2, 0
.save b0, savedbrp
mov savedbrp = brp
.save ar.lc, t22
mov t22 = ar.lc // save ar.lc
mov t7 = 0
mov t8 = 1
PROLOGUE_END
mov ar.lc = 1 // 2 pairs of debug registers
;;
Krdr_Loop:
ld8 t1 = [t11], 16 // get dbr pair
ld8 t2 = [t12], 16 // step by 16 = 1 pair of DRs
ld8 t3 = [t13], 16 // get dbr pair
ld8 t4 = [t14], 16 // step by 16 = 1 pair of DRs
;;
.auto
mov ibr[t7] = t1 // restore ibr pair
mov ibr[t8] = t2
.auto
mov dbr[t7] = t3 // restore dbr pair
mov dbr[t8] = t4
;;
#ifndef NO_12241
srlz.d
#endif
.default
add t7 = 2, t7 // next pair
add t8 = 2, t8
br.cloop.sptk Krdr_Loop
;;
mov ar.lc = t22 // restore ar.lc
(pt2) br.ret.sptk brp // return if loading user
mov out0 = PSR_DB
mov out1 = 1
(pt3) br.call.spnt brp = KeSetLowPsrBit // set psr.db if loading kernel
;;
NESTED_RETURN
;;
NESTED_EXIT(KiLoadKernelDebugRegisters)
//++
//--------------------------------------------------------------------
// Routine:
//
// VOID
// KiSaveExceptionFrame(PKEXCEPTION_FRAME)
//
// Description:
//
// Save preserved context in exception frame.
//
// Note: This routine modifies the UNAT register. The caller
// is resposible for preserving UNAT before calling.
//
// Input:
//
// a0: points to exception frame
//
// Output:
//
// None
//
// Return value:
//
// none
//
// Note: t0 may contain the trap frame address; don't touch it.
//
//--------------------------------------------------------------------
LEAF_ENTRY(KiSaveExceptionFrame)
//
// Local register aliases
//
rpEF1 = t10
rpEF2 = t11
add rpEF1 = ExIntS0, a0 // -> ExIntS0
movl t12 = PFS_EC_MASK << PFS_EC_SHIFT
;;
add rpEF2 = ExIntS1, a0 // -> ExIntS1
mov t3 = ar.pfs
;;
and t3 = t3, t12
.mem.offset 0,0
st8.spill [rpEF1] = s0, ExIntS2-ExIntS0
.mem.offset 8,0
st8.spill [rpEF2] = s1, ExIntS3-ExIntS1
mov t4 = ar.lc
;;
.mem.offset 0,0
st8.spill [rpEF1] = s2, ExApEC-ExIntS2
.mem.offset 8,0
st8.spill [rpEF2] = s3, ExApLC-ExIntS3
mov t5 = bs0
;;
st8 [rpEF1] = t3, ExBrS0-ExApEC
st8 [rpEF2] = t4, ExBrS1-ExApLC
mov t6 = bs1
;;
mov t2 = ar.unat // save user nat register for
mov t7 = bs2
mov t8 = bs3
st8 [rpEF1] = t5, ExBrS2-ExBrS0
st8 [rpEF2] = t6, ExBrS3-ExBrS1
mov t9 = bs4
;;
st8 [rpEF1] = t7, ExBrS4-ExBrS2
st8 [rpEF2] = t8, ExIntNats-ExBrS3
;;
st8 [rpEF1] = t9, ExFltS0-ExBrS4
st8 [rpEF2] = t2, ExFltS1-ExIntNats
;;
stf.spill [rpEF1] = fs0, ExFltS2-ExFltS0
stf.spill [rpEF2] = fs1, ExFltS3-ExFltS1
;;
stf.spill [rpEF1] = fs2, ExFltS4-ExFltS2
stf.spill [rpEF2] = fs3, ExFltS5-ExFltS3
;;
stf.spill [rpEF1] = fs4, ExFltS6-ExFltS4
stf.spill [rpEF2] = fs5, ExFltS7-ExFltS5
;;
stf.spill [rpEF1] = fs6, ExFltS8-ExFltS6
stf.spill [rpEF2] = fs7, ExFltS9-ExFltS7
;;
stf.spill [rpEF1] = fs8, ExFltS10-ExFltS8
stf.spill [rpEF2] = fs9, ExFltS11-ExFltS9
;;
stf.spill [rpEF1] = fs10, ExFltS12-ExFltS10
stf.spill [rpEF2] = fs11, ExFltS13-ExFltS11
;;
stf.spill [rpEF1] = fs12, ExFltS14-ExFltS12
stf.spill [rpEF2] = fs13, ExFltS15-ExFltS13
;;
stf.spill [rpEF1] = fs14, ExFltS16-ExFltS14
stf.spill [rpEF2] = fs15, ExFltS17-ExFltS15
;;
stf.spill [rpEF1] = fs16, ExFltS18-ExFltS16
stf.spill [rpEF2] = fs17, ExFltS19-ExFltS17
;;
stf.spill [rpEF1] = fs18
stf.spill [rpEF2] = fs19
LEAF_RETURN
;;
LEAF_EXIT(KiSaveExceptionFrame)
//--------------------------------------------------------------------
// Routine:
//
// VOID
// KiRestoreExceptionFrame(PKEXCEPTION_FRAME)
//
// Description:
//
// Restores preserved context from the exception frame. Also
// restore volatile part of floating point context not restored with
// rest of volatile context.
//
// Note: This routine does not use v0, t21 or t22. This routine's
// caller may be dependent on these registers (for performance
// in the context switch path).
//
// Note: This routine modifies the UNAT register. The caller
// is resposible for restoring UNAT to the correct value.
//
// Input:
//
// a0: points to exception frame
//
// Output:
//
// None
//
// Return value:
//
// none
//
//--------------------------------------------------------------------
LEAF_ENTRY(KiRestoreExceptionFrame)
add t16 = ExIntNats, a0
movl t12 = PFS_EC_MASK << PFS_EC_SHIFT
add t17 = ExApEC, a0
nop.f 0
mov t13 = ar.pfs
;;
ld8.nta t2 = [t16], ExBrS0-ExIntNats
ld8.nta t3 = [t17], ExApLC-ExApEC
;;
ld8.nta t5 = [t16], ExBrS1-ExBrS0
ld8.nta t4 = [t17], ExBrS2-ExApLC
;;
ld8.nta t6 = [t16], ExBrS3-ExBrS1
ld8.nta t7 = [t17], ExBrS4-ExBrS2
;;
ld8.nta t8 = [t16], ExFltS0-ExBrS3
ld8.nta t9 = [t17], ExFltS1-ExBrS4
;;
ldf.fill.nta fs0 = [t16], ExFltS2-ExFltS0
ldf.fill.nta fs1 = [t17], ExFltS3-ExFltS1
add t18 = ExIntS0, a0
;;
ldf.fill.nta fs2 = [t16], ExFltS4-ExFltS2
ldf.fill.nta fs3 = [t17], ExFltS5-ExFltS3
add t19 = ExIntS1, a0
;;
ldf.fill.nta fs4 = [t16], ExFltS6-ExFltS4
ldf.fill.nta fs5 = [t17], ExFltS7-ExFltS5
andcm t13 = t13, t12 // zero out EC field
;;
ldf.fill.nta fs6 = [t16], ExFltS8-ExFltS6
ldf.fill.nta fs7 = [t17], ExFltS9-ExFltS7
and t3 = t3, t12 // capture EC value
;;
mov ar.unat = t2
or t13 = t3, t13 // deposit into PFS.EC field
;;
ldf.fill.nta fs8 = [t16], ExFltS10-ExFltS8
ldf.fill.nta fs9 = [t17], ExFltS11-ExFltS9
mov ar.pfs = t13
;;
ldf.fill.nta fs10 = [t16], ExFltS12-ExFltS10
ldf.fill.nta fs11 = [t17], ExFltS13-ExFltS11
mov ar.lc = t4
;;
ldf.fill.nta fs12 = [t16], ExFltS14-ExFltS12
ldf.fill.nta fs13 = [t17], ExFltS15-ExFltS13
mov bs0 = t5
;;
ldf.fill.nta fs14 = [t16], ExFltS16-ExFltS14
ldf.fill.nta fs15 = [t17], ExFltS17-ExFltS15
mov bs1 = t6
;;
ldf.fill.nta fs16 = [t16], ExFltS18-ExFltS16
ldf.fill.nta fs17 = [t17], ExFltS19-ExFltS17
mov bs2 = t7
;;
ldf.fill.nta fs18 = [t16]
ldf.fill.nta fs19 = [t17]
mov bs3 = t8
ld8.fill.nta s0 = [t18], ExIntS2-ExIntS0
ld8.fill.nta s1 = [t19], ExIntS3-ExIntS1
mov bs4 = t9
;;
ld8.fill.nta s2 = [t18]
ld8.fill.nta s3 = [t19]
LEAF_RETURN
;;
LEAF_EXIT(KiRestoreExceptionFrame)
//++
//--------------------------------------------------------------------
// Routine:
//
// KiSaveHigherFPVolatile(PKHIGHER_FP_SAVEAREA)
//
// Description:
//
// Save higher FP volatile context in higher FP save area
//
// Input:
//
// a0: pointer to higher FP save area
// brp: return address
//
// Output:
//
// None
//
// Return value:
//
// None
//
//--------------------------------------------------------------------
NESTED_ENTRY(KiSaveHigherFPVolatile)
NESTED_SETUP(1, 3, 1, 0)
PROLOGUE_END
//
// Local register aliases
//
rpSA1 = t0
rpSA2 = t1
//
// Clear DFH bit so the high floating point set may be saved by the kernel
// Disable interrupts so that save is atomic
//
GET_IRQL (loc2)
mov t2 = psr.um
movl t3 = KiPcr+PcCurrentThread
;;
cmp.ge pt2, pt3 = APC_LEVEL, loc2
;;
PSET_IRQL (pt2, DISPATCH_LEVEL)
ld8 t4 = [t3], PcHighFpOwner-PcCurrentThread
;;
ld8 t5 = [t3]
;;
cmp.ne pt1 = t4, t5
(pt1) br.cond.spnt Kshfpv20
;;
tbit.z pt0 = t2, PSR_MFH
(pt0) br.cond.spnt Kshfpv20
br Kshfpv10
;;
ALTERNATE_ENTRY(KiSaveHigherFPVolatileAtDispatchLevel)
NESTED_SETUP(1, 3, 1, 0)
cmp.ne pt2, pt3 = r0, r0 // set pt2 to FALSE, pt3 to TRUE
PROLOGUE_END
movl t3 = KiPcr+PcCurrentThread
;;
ld8 t4 = [t3], PcHighFpOwner-PcCurrentThread
;;
ld8 t5 = [t3]
;;
cmp.ne pt1 = t4, t5
;;
(pt1) break.i BREAKPOINT_STOP
Kshfpv10:
rsm (1 << PSR_DFH)
add rpSA1 = HiFltF32, a0 // -> HiFltF32
add rpSA2 = HiFltF33, a0 // -> HiFltF33
;;
srlz.d
stf.spill.nta [rpSA1] = f32, HiFltF34-HiFltF32
stf.spill.nta [rpSA2] = f33, HiFltF35-HiFltF33
;;
stf.spill.nta [rpSA1] = f34, HiFltF36-HiFltF34
stf.spill.nta [rpSA2] = f35, HiFltF37-HiFltF35
;;
stf.spill.nta [rpSA1] = f36, HiFltF38-HiFltF36
stf.spill.nta [rpSA2] = f37, HiFltF39-HiFltF37
;;
stf.spill.nta [rpSA1] = f38, HiFltF40-HiFltF38
stf.spill.nta [rpSA2] = f39, HiFltF41-HiFltF39
;;
stf.spill.nta [rpSA1] = f40, HiFltF42-HiFltF40
stf.spill.nta [rpSA2] = f41, HiFltF43-HiFltF41
;;
stf.spill.nta [rpSA1] = f42, HiFltF44-HiFltF42
stf.spill.nta [rpSA2] = f43, HiFltF45-HiFltF43
;;
stf.spill.nta [rpSA1] = f44, HiFltF46-HiFltF44
stf.spill.nta [rpSA2] = f45, HiFltF47-HiFltF45
;;
stf.spill.nta [rpSA1] = f46, HiFltF48-HiFltF46
stf.spill.nta [rpSA2] = f47, HiFltF49-HiFltF47
;;
stf.spill.nta [rpSA1] = f48, HiFltF50-HiFltF48
stf.spill.nta [rpSA2] = f49, HiFltF51-HiFltF49
;;
stf.spill.nta [rpSA1] = f50, HiFltF52-HiFltF50
stf.spill.nta [rpSA2] = f51, HiFltF53-HiFltF51
;;
stf.spill.nta [rpSA1] = f52, HiFltF54-HiFltF52
stf.spill.nta [rpSA2] = f53, HiFltF55-HiFltF53
;;
stf.spill.nta [rpSA1] = f54, HiFltF56-HiFltF54
stf.spill.nta [rpSA2] = f55, HiFltF57-HiFltF55
;;
stf.spill.nta [rpSA1] = f56, HiFltF58-HiFltF56
stf.spill.nta [rpSA2] = f57, HiFltF59-HiFltF57
;;
stf.spill.nta [rpSA1] = f58, HiFltF60-HiFltF58
stf.spill.nta [rpSA2] = f59, HiFltF61-HiFltF59
;;
stf.spill.nta [rpSA1] = f60, HiFltF62-HiFltF60
stf.spill.nta [rpSA2] = f61, HiFltF63-HiFltF61
;;
stf.spill.nta [rpSA1] = f62, HiFltF64-HiFltF62
stf.spill.nta [rpSA2] = f63, HiFltF65-HiFltF63
;;
stf.spill.nta [rpSA1] = f64, HiFltF66-HiFltF64
stf.spill.nta [rpSA2] = f65, HiFltF67-HiFltF65
;;
stf.spill.nta [rpSA1] = f66, HiFltF68-HiFltF66
stf.spill.nta [rpSA2] = f67, HiFltF69-HiFltF67
;;
stf.spill.nta [rpSA1] = f68, HiFltF70-HiFltF68
stf.spill.nta [rpSA2] = f69, HiFltF71-HiFltF69
;;
stf.spill.nta [rpSA1] = f70, HiFltF72-HiFltF70
stf.spill.nta [rpSA2] = f71, HiFltF73-HiFltF71
;;
stf.spill.nta [rpSA1] = f72, HiFltF74-HiFltF72
stf.spill.nta [rpSA2] = f73, HiFltF75-HiFltF73
;;
stf.spill.nta [rpSA1] = f74, HiFltF76-HiFltF74
stf.spill.nta [rpSA2] = f75, HiFltF77-HiFltF75
;;
stf.spill.nta [rpSA1] = f76, HiFltF78-HiFltF76
stf.spill.nta [rpSA2] = f77, HiFltF79-HiFltF77
;;
stf.spill.nta [rpSA1] = f78, HiFltF80-HiFltF78
stf.spill.nta [rpSA2] = f79, HiFltF81-HiFltF79
;;
stf.spill.nta [rpSA1] = f80, HiFltF82-HiFltF80
stf.spill.nta [rpSA2] = f81, HiFltF83-HiFltF81
;;
stf.spill.nta [rpSA1] = f82, HiFltF84-HiFltF82
stf.spill.nta [rpSA2] = f83, HiFltF85-HiFltF83
;;
stf.spill.nta [rpSA1] = f84, HiFltF86-HiFltF84
stf.spill.nta [rpSA2] = f85, HiFltF87-HiFltF85
;;
stf.spill.nta [rpSA1] = f86, HiFltF88-HiFltF86
stf.spill.nta [rpSA2] = f87, HiFltF89-HiFltF87
;;
stf.spill.nta [rpSA1] = f88, HiFltF90-HiFltF88
stf.spill.nta [rpSA2] = f89, HiFltF91-HiFltF89
;;
stf.spill.nta [rpSA1] = f90, HiFltF92-HiFltF90
stf.spill.nta [rpSA2] = f91, HiFltF93-HiFltF91
;;
stf.spill.nta [rpSA1] = f92, HiFltF94-HiFltF92
stf.spill.nta [rpSA2] = f93, HiFltF95-HiFltF93
;;
stf.spill.nta [rpSA1] = f94, HiFltF96-HiFltF94
stf.spill.nta [rpSA2] = f95, HiFltF97-HiFltF95
;;
stf.spill.nta [rpSA1] = f96, HiFltF98-HiFltF96
stf.spill.nta [rpSA2] = f97, HiFltF99-HiFltF97
;;
stf.spill.nta [rpSA1] = f98, HiFltF100-HiFltF98
stf.spill.nta [rpSA2] = f99, HiFltF101-HiFltF99
;;
stf.spill.nta [rpSA1] = f100, HiFltF102-HiFltF100
stf.spill.nta [rpSA2] = f101, HiFltF103-HiFltF101
;;
stf.spill.nta [rpSA1] = f102, HiFltF104-HiFltF102
stf.spill.nta [rpSA2] = f103, HiFltF105-HiFltF103
;;
stf.spill.nta [rpSA1] = f104, HiFltF106-HiFltF104
stf.spill.nta [rpSA2] = f105, HiFltF107-HiFltF105
;;
stf.spill.nta [rpSA1] = f106, HiFltF108-HiFltF106
stf.spill.nta [rpSA2] = f107, HiFltF109-HiFltF107
;;
stf.spill.nta [rpSA1] = f108, HiFltF110-HiFltF108
stf.spill.nta [rpSA2] = f109, HiFltF111-HiFltF109
;;
stf.spill.nta [rpSA1] = f110, HiFltF112-HiFltF110
stf.spill.nta [rpSA2] = f111, HiFltF113-HiFltF111
;;
stf.spill.nta [rpSA1] = f112, HiFltF114-HiFltF112
stf.spill.nta [rpSA2] = f113, HiFltF115-HiFltF113
;;
stf.spill.nta [rpSA1] = f114, HiFltF116-HiFltF114
stf.spill.nta [rpSA2] = f115, HiFltF117-HiFltF115
;;
stf.spill.nta [rpSA1] = f116, HiFltF118-HiFltF116
stf.spill.nta [rpSA2] = f117, HiFltF119-HiFltF117
;;
stf.spill.nta [rpSA1] = f118, HiFltF120-HiFltF118
stf.spill.nta [rpSA2] = f119, HiFltF121-HiFltF119
;;
stf.spill.nta [rpSA1] = f120, HiFltF122-HiFltF120
stf.spill.nta [rpSA2] = f121, HiFltF123-HiFltF121
;;
stf.spill.nta [rpSA1] = f122, HiFltF124-HiFltF122
stf.spill.nta [rpSA2] = f123, HiFltF125-HiFltF123
;;
stf.spill.nta [rpSA1] = f124, HiFltF126-HiFltF124
stf.spill.nta [rpSA2] = f125, HiFltF127-HiFltF125
;;
stf.spill.nta [rpSA1] = f126
stf.spill.nta [rpSA2] = f127
//
// Set DFH bit so the high floating point set may not be used by the kernel
// Must clear mfh after fp registers saved
//
Kshfpv20:
ssm 1 << PSR_DFH
;;
rum 1 << PSR_MFH
(pt3) br.ret.sptk brp
LOWER_IRQL(loc2)
NESTED_RETURN
;;
LEAF_EXIT(KiSaveHigherFPVolatile)
//++
//--------------------------------------------------------------------
// Routine:
//
// KiRestoreHigherFPVolatile()
//
// Description:
//
// Restore higher FP volatile context from higher FP save area
//
// N.B. This function is carefully constructed to use only scratch
// registers rHpT1, rHpT3, and rTH2. This function may be
// called by C code and the disabled fp vector when user
// and kernel bank is used respectively.
// N.B. Caller must ensure higher fp enabled (psr.dfh=0)
// N.B. Caller must ensure no interrupt during restore
//
// Input:
//
// None.
//
// Output:
//
// None
//
// Return value:
//
// None
//
//--------------------------------------------------------------------
LEAF_ENTRY(KiRestoreHigherFPVolatile)
movl rHpT1 = KiPcr+PcCurrentThread
;;
ld8 rHpT3 = [rHpT1], PcHighFpOwner-PcCurrentThread
;;
st8 [rHpT1] = rHpT3, PcNumber-PcHighFpOwner
add rHpT3 = ThNumber, rHpT3
;;
ld1 rHpT1 = [rHpT1] // load processor #
;;
st1 [rHpT3] = rHpT1 // save it in Thread->Number
add rHpT3 = ThStackBase-ThNumber, rHpT3
;;
ld8 rHpT3 = [rHpT3] // load kernel stack base
;;
add rHpT1 = -ThreadStateSaveAreaLength+TsHigherFPVolatile+HiFltF32, rHpT3
add rHpT3 = -ThreadStateSaveAreaLength+TsHigherFPVolatile+HiFltF33, rHpT3
;;
ldf.fill.nta f32 = [rHpT1], HiFltF34-HiFltF32
ldf.fill.nta f33 = [rHpT3], HiFltF35-HiFltF33
;;
ldf.fill.nta f34 = [rHpT1], HiFltF36-HiFltF34
ldf.fill.nta f35 = [rHpT3], HiFltF37-HiFltF35
;;
ldf.fill.nta f36 = [rHpT1], HiFltF38-HiFltF36
ldf.fill.nta f37 = [rHpT3], HiFltF39-HiFltF37
;;
ldf.fill.nta f38 = [rHpT1], HiFltF40-HiFltF38
ldf.fill.nta f39 = [rHpT3], HiFltF41-HiFltF39
;;
ldf.fill.nta f40 = [rHpT1], HiFltF42-HiFltF40
ldf.fill.nta f41 = [rHpT3], HiFltF43-HiFltF41
;;
ldf.fill.nta f42 = [rHpT1], HiFltF44-HiFltF42
ldf.fill.nta f43 = [rHpT3], HiFltF45-HiFltF43
;;
ldf.fill.nta f44 = [rHpT1], HiFltF46-HiFltF44
ldf.fill.nta f45 = [rHpT3], HiFltF47-HiFltF45
;;
ldf.fill.nta f46 = [rHpT1], HiFltF48-HiFltF46
ldf.fill.nta f47 = [rHpT3], HiFltF49-HiFltF47
;;
ldf.fill.nta f48 = [rHpT1], HiFltF50-HiFltF48
ldf.fill.nta f49 = [rHpT3], HiFltF51-HiFltF49
;;
ldf.fill.nta f50 = [rHpT1], HiFltF52-HiFltF50
ldf.fill.nta f51 = [rHpT3], HiFltF53-HiFltF51
;;
ldf.fill.nta f52 = [rHpT1], HiFltF54-HiFltF52
ldf.fill.nta f53 = [rHpT3], HiFltF55-HiFltF53
;;
ldf.fill.nta f54 = [rHpT1], HiFltF56-HiFltF54
ldf.fill.nta f55 = [rHpT3], HiFltF57-HiFltF55
;;
ldf.fill.nta f56 = [rHpT1], HiFltF58-HiFltF56
ldf.fill.nta f57 = [rHpT3], HiFltF59-HiFltF57
;;
ldf.fill.nta f58 = [rHpT1], HiFltF60-HiFltF58
ldf.fill.nta f59 = [rHpT3], HiFltF61-HiFltF59
;;
ldf.fill.nta f60 = [rHpT1], HiFltF62-HiFltF60
ldf.fill.nta f61 = [rHpT3], HiFltF63-HiFltF61
;;
ldf.fill.nta f62 = [rHpT1], HiFltF64-HiFltF62
ldf.fill.nta f63 = [rHpT3], HiFltF65-HiFltF63
;;
ldf.fill.nta f64 = [rHpT1], HiFltF66-HiFltF64
ldf.fill.nta f65 = [rHpT3], HiFltF67-HiFltF65
;;
ldf.fill.nta f66 = [rHpT1], HiFltF68-HiFltF66
ldf.fill.nta f67 = [rHpT3], HiFltF69-HiFltF67
;;
ldf.fill.nta f68 = [rHpT1], HiFltF70-HiFltF68
ldf.fill.nta f69 = [rHpT3], HiFltF71-HiFltF69
;;
ldf.fill.nta f70 = [rHpT1], HiFltF72-HiFltF70
ldf.fill.nta f71 = [rHpT3], HiFltF73-HiFltF71
;;
ldf.fill.nta f72 = [rHpT1], HiFltF74-HiFltF72
ldf.fill.nta f73 = [rHpT3], HiFltF75-HiFltF73
;;
ldf.fill.nta f74 = [rHpT1], HiFltF76-HiFltF74
ldf.fill.nta f75 = [rHpT3], HiFltF77-HiFltF75
;;
ldf.fill.nta f76 = [rHpT1], HiFltF78-HiFltF76
ldf.fill.nta f77 = [rHpT3], HiFltF79-HiFltF77
;;
ldf.fill.nta f78 = [rHpT1], HiFltF80-HiFltF78
ldf.fill.nta f79 = [rHpT3], HiFltF81-HiFltF79
;;
ldf.fill.nta f80 = [rHpT1], HiFltF82-HiFltF80
ldf.fill.nta f81 = [rHpT3], HiFltF83-HiFltF81
;;
ldf.fill.nta f82 = [rHpT1], HiFltF84-HiFltF82
ldf.fill.nta f83 = [rHpT3], HiFltF85-HiFltF83
;;
ldf.fill.nta f84 = [rHpT1], HiFltF86-HiFltF84
ldf.fill.nta f85 = [rHpT3], HiFltF87-HiFltF85
;;
ldf.fill.nta f86 = [rHpT1], HiFltF88-HiFltF86
ldf.fill.nta f87 = [rHpT3], HiFltF89-HiFltF87
;;
ldf.fill.nta f88 = [rHpT1], HiFltF90-HiFltF88
ldf.fill.nta f89 = [rHpT3], HiFltF91-HiFltF89
;;
ldf.fill.nta f90 = [rHpT1], HiFltF92-HiFltF90
ldf.fill.nta f91 = [rHpT3], HiFltF93-HiFltF91
;;
ldf.fill.nta f92 = [rHpT1], HiFltF94-HiFltF92
ldf.fill.nta f93 = [rHpT3], HiFltF95-HiFltF93
;;
ldf.fill.nta f94 = [rHpT1], HiFltF96-HiFltF94
ldf.fill.nta f95 = [rHpT3], HiFltF97-HiFltF95
;;
ldf.fill.nta f96 = [rHpT1], HiFltF98-HiFltF96
ldf.fill.nta f97 = [rHpT3], HiFltF99-HiFltF97
;;
ldf.fill.nta f98 = [rHpT1], HiFltF100-HiFltF98
ldf.fill.nta f99 = [rHpT3], HiFltF101-HiFltF99
;;
ldf.fill.nta f100 = [rHpT1], HiFltF102-HiFltF100
ldf.fill.nta f101 = [rHpT3], HiFltF103-HiFltF101
;;
ldf.fill.nta f102 = [rHpT1], HiFltF104-HiFltF102
ldf.fill.nta f103 = [rHpT3], HiFltF105-HiFltF103
;;
ldf.fill.nta f104 = [rHpT1], HiFltF106-HiFltF104
ldf.fill.nta f105 = [rHpT3], HiFltF107-HiFltF105
;;
ldf.fill.nta f106 = [rHpT1], HiFltF108-HiFltF106
ldf.fill.nta f107 = [rHpT3], HiFltF109-HiFltF107
;;
ldf.fill.nta f108 = [rHpT1], HiFltF110-HiFltF108
ldf.fill.nta f109 = [rHpT3], HiFltF111-HiFltF109
;;
ldf.fill.nta f110 = [rHpT1], HiFltF112-HiFltF110
ldf.fill.nta f111 = [rHpT3], HiFltF113-HiFltF111
;;
ldf.fill.nta f112 = [rHpT1], HiFltF114-HiFltF112
ldf.fill.nta f113 = [rHpT3], HiFltF115-HiFltF113
;;
ldf.fill.nta f114 = [rHpT1], HiFltF116-HiFltF114
ldf.fill.nta f115 = [rHpT3], HiFltF117-HiFltF115
;;
ldf.fill.nta f116 = [rHpT1], HiFltF118-HiFltF116
ldf.fill.nta f117 = [rHpT3], HiFltF119-HiFltF117
;;
ldf.fill.nta f118 = [rHpT1], HiFltF120-HiFltF118
ldf.fill.nta f119 = [rHpT3], HiFltF121-HiFltF119
;;
ldf.fill.nta f120 = [rHpT1], HiFltF122-HiFltF120
ldf.fill.nta f121 = [rHpT3], HiFltF123-HiFltF121
;;
ldf.fill.nta f122 = [rHpT1], HiFltF124-HiFltF122
ldf.fill.nta f123 = [rHpT3], HiFltF125-HiFltF123
;;
ldf.fill.nta f124 = [rHpT1], HiFltF126-HiFltF124
ldf.fill.nta f125 = [rHpT3], HiFltF127-HiFltF125
;;
ldf.fill.nta f126 = [rHpT1]
ldf.fill.nta f127 = [rHpT3]
;;
rsm 1 << PSR_MFH // clear psr.mfh bit
br.ret.sptk brp
;;
LEAF_EXIT(KiRestoreHigherFPVolatile)
//
// ++
//
// Routine:
//
// KiPageTableFault
//
// Description:
//
// Branched from Inst/DataTlbVector
// Inserts a missing PDE translation for VHPT mapping
// If PageNotPresent-bit of PDE is not set,
// branches out to KiPageFault
//
// On entry:
//
// rva (h24) : offending virtual address
// riha (h25) : a offending PTE address
// rpr: (h26) : saved predicate
//
// Handle:
//
// Extracts the PDE index from riha (PTE address in VHPT) and
// generates a PDE address by adding to VHPT_DIRBASE. When accesses
// a page directory entry (PDE), there might be a TLB miss on the
// page directory table and returns a NaT on ld8.s. If so, branches
// to KiPageDirectoryTableFault. If the page-not-present bit of the
// PDE is not set, branches to KiPageNotPresentFault. Otherwise,
// inserts the PDE entry into the data TC (translation cache).
//
// Notes:
//
//
// --
HANDLER_ENTRY(KiPageTableFault)
rva = h24
riha = h25
rpr = h26
rpPde = h27
rPde = h28
rPde2 = h29
rps = h30
rISR = h31
thash rpPde = riha // M
cmp.ne pt1 = r0, r0
mov rps = PAGE_SHIFT << PS_SHIFT // I
;;
mov cr.itir = rps // M
ld8.s rPde = [rpPde] // M, load PDE
;;
extr.u rPde2 = rPde, 2, 3 // I
tnat.nz pt0, p0 = rPde // I
tbit.z.or pt1, p0 = rPde, PTE_ACCESS
tbit.z.or pt1, p0 = rPde, PTE_VALID // I, if non-present page fault
(pt0) br.cond.spnt KiPageDirectoryFault // B, tb miss on PDE access
(pt1) br.cond.spnt KiPageFault // B, page fault
;;
cmp.eq pt3 = 1, rPde2 // A
mov cr.ifa = riha // M
(pt3) br.cond.spnt KiLargePage
;;
itc.d rPde // M
;;
#if !defined(NT_UP)
ld8.s rPde2 = [rpPde] // M
cmp.ne pt0 = zero, zero // I
;;
cmp.ne.or pt0, p0 = rPde2, rPde // I, if PTEs are different
tnat.nz.or pt0, p0 = rPde2 // I
;;
(pt0) ptc.l riha, rps // M, purge it
#endif
mov pr = rpr, -1 // I
rfi;; // B
//
// This is a large page PDE entry.
//
KiLargePage:
mov cr.ifa = rva // M
mov rISR = cr.isr
extr.u rPde2 = rPde, PTE_LP_CACHE_SHIFT, PTE_CACHE_LEN
;;
mov rps = LARGE_PAGE_SHIFT << PS_SHIFT
dep rPde2 = rPde2, rPde, 2, 3 // Set the cachable attribute.
;;
mov cr.itir = rps // M
tbit.z pt3, pt4 = rISR, ISR_X // I
;;
(pt3) itc.d rPde2 // M
;;
(pt4) itc.i rPde2 // M
;;
#if !defined(NT_UP)
ld8.s rPde2 = [rpPde] // M
cmp.ne pt0 = zero, zero // I
;;
cmp.ne.or pt0, p0 = rPde2, rPde // I, if PTEs are different
tnat.nz.or pt0, p0 = rPde2 // I
;;
(pt0) ptc.l rva, rps // M, purge it
#endif
mov pr = rpr, -1 // I
rfi;; // B
HANDLER_EXIT(KiPageTableFault)
//++
//
// KiPageDirectoryFault
//
// Cause:
//
// Parameters:
// rpPde (h28) : pointer to PDE entry
// rpr (h26) : saved predicate
//
//
// Handle:
//
//--
HANDLER_ENTRY(KiPageDirectoryFault)
rva = h24
rpPpe = h25
rpr = h26
rpPde = h27
rPpe = h28
rPpe2 = h29
rps = h30
thash rpPpe = rpPde // M
cmp.ne pt0 = r0, r0
;;
ld8.s rPpe = [rpPpe] // M
;;
tnat.nz.or pt0, p0 = rPpe // I
tbit.z.or pt0, p0 = rPpe, PTE_ACCESS
tbit.z.or pt0, p0 = rPpe, PTE_VALID // I, if non-present page fault
(pt0) br.cond.spnt KiPageFault // B
;;
mov cr.ifa = rpPde // M, set tva for vhpt translation
;;
itc.d rPpe // M
;;
#if !defined(NT_UP)
ld8.s rPpe2 = [rpPpe] // M
mov rps = PAGE_SHIFT << PS_SHIFT // I
cmp.ne pt0 = zero, zero // I
;;
cmp.ne.or pt0, p0 = rPpe2, rPpe // I, if PTEs are different
tnat.nz.or pt0, p0 = rPpe2 // I
;;
(pt0) ptc.l rpPde, rps // M, purge it
#endif
mov pr = rpr, -1 // I
rfi;; // B
HANDLER_EXIT(KiPageDirectoryFault)
//
// ++
//
// Routine:
//
// KiPteNotPresentFault
//
// Description:
//
// Branched from KiVhptTransVector and KiPageTableFault.
// Inserts a missing PDE translation for VHPT mapping
// If no PDE for it, branches out to KiPageFault
//
// On entry:
//
// rva (h24) : offending virtual address
// rpr (h26) : saved predicate
// rPde (h28) : PDE entry
//
// Handle:
//
// Check to see if PDE is marked as LARGE_PAGE. If so,
// make it valid and install the large page size PTE.
// If not, branch to KiPageFault.
//
//
// Notes:
//
// PCR page mapped with TR
// --
HANDLER_ENTRY(KiPteNotPresentFault)
rva = h24 // passed
riha = h25 // passed
rpr = h26 // passed
rps = h27
rPfn = h28
rpAte = h28
rAte = h29
rAteEnd = h30
rAteBase = h31
rAteMask = h22
pIndr = pt1
mov rps = PS_4K << PS_SHIFT // M
movl rAteBase = ALT4KB_BASE // L
mov rAteMask = ATE_MASK0 // I
shr.u rPfn = rva, PAGE4K_SHIFT // I
;;
shladd rpAte = rPfn, PTE_SHIFT, rAteBase // M
movl rAteEnd = ALT4KB_END // L
;;
ld8.s rAte = [rpAte] // M
andcm rAteMask = -1, rAteMask // I
cmp.ltu pIndr = rpAte, rAteEnd // I
;;
tnat.z.and pIndr = rAte // I
tbit.nz.and pIndr = rAte, PTE_VALID // I
or rAteMask = rAte, rAteMask // M
tbit.nz.and pIndr = rAte, PTE_ACCESS // I
tbit.nz.and pIndr = rAte, ATE_INDIRECT // I
(pIndr) br.cond.spnt KiPteIndirectFault
br.sptk KiPageFault // B
HANDLER_EXIT(KiPteNotPresentFault)
//
// ++
//
// Routine:
//
// KiKseg4Fault
//
// Description:
//
// TLB miss on KSEG4 space
//
//
// On entry:
//
// rva (h24) : faulting virtual address
// riha (h25) : IHA address
// rpr (h26) : saved predicate
//
// Process:
//
//
// Notes:
//
// PCR page mapped with TR
// --
HANDLER_ENTRY(KiKseg4Fault)
rIPSR = h22
rISR = h23
rva = h24 // passed
riha = h25
rpr = h26 // passed
rPte = h27
mov rISR = cr.isr // M
movl rPte = VALID_KERNEL_PTE | PTE_NOCACHE // L
mov rIPSR = cr.ipsr // M
shr.u rva = rva, PAGE_SHIFT // I
;;
tbit.z pt2, pt3 = rISR, ISR_SP // I
dep.z rva = rva, PAGE_SHIFT, 32 // I
;;
or rPte = rPte, rva // I
dep rIPSR = 1, rIPSR, PSR_ED, 1 // I
;;
(pt2) itc.d rPte // M
;;
(pt3) mov cr.ipsr = rIPSR // M
;;
mov pr = rpr, -1 // I
rfi // B
;;
HANDLER_EXIT(KiKseg4Fault)
//
// ++
//
// Routine:
//
// KiPageFault
//
// Description:
//
// This must be a genuine page fault. Call KiMemoryFault().
//
//
// On entry:
//
// rva (h24) : offending virtual address
// rpr (h26) : PDE contents
//
// Process:
//
// Restores the save predicate (pr), and branches to
// KiGenericExceptionHandler with the argument KiMemoryFault with
// macro VECTOR_CALL_HANDLER().
//
// Notes:
//
// PCR page mapped with TR
// --
HANDLER_ENTRY(KiPageFault)
rva = h24
rPSR_ED = h25
rpr = h26
rIPSR = h27
rps = h29
rCode = h30
rISR = h31
//
// check to see if a page fault occurred on a speculative load or
// lfetch.fault. if so, set IPSR.ed bit. This forces to generate a NaT on
// ld.s and steps out to the next instruction on lfetch.fault after
// rfi.
//
mov rISR = cr.isr // M
movl rPSR_ED = 1 << PSR_ED // L
mov rIPSR = cr.ipsr // M
;;
and rCode = ISR_NA_CODE_MASK, rISR // A
or rIPSR = rIPSR, rPSR_ED // A
tbit.nz pt1, pt2 = rISR, ISR_NA // I
;;
(pt1) cmp.ne pt0, p0 = ISR_LFETCH, rCode // A
(pt2) tbit.z pt0, p0 = rISR, ISR_SP // I
(pt0) br.cond.spnt KiCallMemoryFault // B
mov cr.ipsr = rIPSR // M
;;
mov pr = rpr, -1 // I
rfi // B
;;
KiCallMemoryFault:
mov rHIFA = cr.ifa
movl rHHandler = KiMemoryFault
mov rps = PAGE_SHIFT << PS_SHIFT
mov pr = rpr, -1 // I
;;
ptc.l rHIFA, rps
br.sptk KiGenericExceptionHandler
HANDLER_EXIT(KiPageFault)
//
// ++
//
// Routine:
//
// KiPteIndirectFault
//
// Description:
//
// The PTE itself indicates a PteIndirect fault. The target PTE address
// should be generated by extracting PteOffset from PTE and adding it to
// PTE_UBASE. The owner field of the target PTE must be 1. Otherwise,
// Call MmX86Fault().
//
// On entry:
//
// rva (h24) : offending virtual address
// rpr (h26) : PDE contents
//
// Process:
//
// Restores the save predicate (pr), and branches to
// KiGenericExceptionHandler with the argument KiMemoryFault with
// macro VECTOR_CALL_HANDLER().
//
// Notes:
//
// PCR page mapped with TR
// --
HANDLER_ENTRY(KiPteIndirectFault)
rpr = h26 // passed
rps = h27 // passed
rpAte = h28 // passed
rAte = h29 // passed
rPte = h30
rPte0 = h31
rAteMask = h22 // passed
rVa12 = h23
rPteOffset = h23
rpNewPte = h21
rOldIIP = h17 // preserved
rIA32IIP = h18
rpVa = h19 // preserved
rIndex = h20 // preserved
rpBuffer= h16
rpPte = h22
pBr = pt0
pPrg = pt3
pLoop = pt4
pClear = pt5
mov cr.itir = rps // M
thash rpNewPte = r0 // M
mov rIA32IIP = cr.iip // M
extr.u rPteOffset = rAte, PAGE4K_SHIFT, 32 // I
;;
add rpNewPte = rPteOffset, rpNewPte // M/I
cmp.eq pLoop, pClear = rIA32IIP, rOldIIP // I
;;
ld8.s rPte = [rpNewPte] // M
shr rVa12 = rpAte, PTE_SHIFT // I
;;
tnat.nz.or pBr = rPte // I
tbit.z.or pBr = rPte, PTE_VALID // I
tbit.z.or pBr = rPte, PTE_ACCESS // I
(pBr) br.cond.spnt KiPageFault
;;
(pClear)mov rIndex = 0 // M
and rPte0 = rPte, rAteMask // I
;;
#if 1
//
// try to acquire the spinlock
// if failed, resume execution and try again on the next fault
//
mov rpVa = 1
movl rpBuffer = KiPcr + PcFpbLock // L
;;
xchg8 rpVa = [rpBuffer], rpVa
;;
cmp.ne pt1 = 0, rpVa
(pt1) br.cond.spnt Kpif10
;;
#endif
//
// deposit extra PFN bits for 4k page
//
dep rPte0 = rVa12, rPte0, PAGE4K_SHIFT, PAGE_SHIFT-PAGE4K_SHIFT // I
;;
(pClear)itc.d rPte0 // M
;;
and rIndex = 7, rIndex // A
movl rpBuffer = KiPcr + PcForwardProgressBuffer // L
;;
shladd rpVa = rIndex, 4, rpBuffer // M
add rIndex = 1, rIndex // I
;;
st8 [rpVa] = rva // M
add rpPte = 8, rpVa // I
;;
st8 [rpPte] = rPte0 // M
#if 1
//
// done updating the forward progress buffer entry, release spinlock
//
movl rpBuffer = KiPcr + PcFpbLock // L
;;
st8.rel [rpBuffer] = r0, PcForwardProgressBuffer - PcFpbLock
#endif
mov rOldIIP = rIA32IIP // I
#if !defined(NT_UP)
rAte2 = h28
rPte2 = h31
ld8.s rPte2 = [rpNewPte] // M
ld8.s rAte2 = [rpAte] // M
cmp.ne pPrg = zero, zero // I
;;
cmp.ne.or pPrg = rPte, rPte2 // I
tnat.nz.or pPrg = rPte2 // I
cmp.ne.or pPrg = rAte, rAte2 // I
tnat.nz.or pPrg = rAte2 // I
;;
(pPrg) ptc.l rva, rps // M
(pPrg) st8 [rpPte] = r0 // M
#endif
(pLoop) br.cond.spnt KiFillForwardProgressTb // B
Kpif10:
mov pr = rpr, -1 // I
rfi;; // B
HANDLER_EXIT(KiPteIndirectFault)
//
// ++
//
// Routine:
//
// Ki4KDataTlbFault
//
// Description:
//
// Branched from KiDataTlbVector if PTE.Cache indicates the reserved
// encoding. Reads the corresponding ATE and creates a 4kb TB on the
// fly inserts it to the TLB. If a looping condition at IIP is
// detected, it branches to KiFillForwardProgressTb and insert the TBs
// from the forward progress TB queue.
//
// On entry:
//
// rva (h24) : offending virtual address
// riha(h25) : IHA address
// rpr (h26) : PDE contents
//
// Notes:
//
// --
HANDLER_ENTRY(Ki4KDataTlbFault)
rva = h24 // passed
riha = h25 // passed
rpr = h26 // passed
rps = h27
rPfn = h28
rpAte = h28
rAte = h29
rPte = h30
rAltBase = h31
rPte0 = h31
rAteMask = h22
rVa12 = h23
rOldIIP = h17 // preserved
rIA32IIP = h18
rpVa = h19
rIndex = h20 // preserved
rpBuffer= h16
rpPte = h21
pBr = pt0
pIndr = pt1
pMiss = pt2
pPrg = pt3
pLoop = pt4
pClear = pt5
pMiss2 = pt6
mov rIA32IIP = cr.iip // M
mov rps = PS_4K << PS_SHIFT // I
dep rAteMask = 1, r0, 32, 1 // Set rAteMask equal to 2**32
;;
cmp.ge pBr, pt7 = rva, rAteMask // M/I, Test to see if rva is out of bounds.
movl rAltBase = ALT4KB_BASE // L
ld8.s rPte = [riha] // M
shr.u rPfn = rva, PAGE4K_SHIFT // I
;;
shladd rpAte = rPfn, PTE_SHIFT, rAltBase // I
;;
ld8.s rAte = [rpAte] // M
movl rAteMask = ATE_MASK // L
;;
cmp.eq pLoop, pClear = rIA32IIP, rOldIIP// M
tnat.nz pMiss = rPte // I
(pMiss) br.cond.spnt KiPageTableFault // B
(pt7) tnat.nz pMiss2 = rAte // I
(pMiss2)br.cond.spnt KiAltTableFault // B
tbit.z.or pBr = rPte, PTE_VALID // I
tbit.z.or pBr = rAte, PTE_VALID // I
tbit.z.or pBr = rAte, PTE_ACCESS // I
or rAteMask = rAte, rAteMask // M
tbit.nz pIndr, p0 = rAte, ATE_INDIRECT // I
(pBr) br.cond.spnt KiPageFault // B
dep rPte0 = 0, rPte, 2, 3 // I, make it WB
shr rVa12 = rpAte, PTE_SHIFT // I
(pIndr) br.cond.spnt KiPteIndirectFault // B
;;
(pClear)mov rIndex = 0 // M
mov cr.itir = rps // M
and rPte0 = rPte0, rAteMask // I
;;
#if 1
//
// try to acquire the spinlock
// if failed, resume execution and try again on the next fault
//
mov rpVa = 1
movl rpBuffer = KiPcr + PcFpbLock // L
;;
xchg8 rpVa = [rpBuffer], rpVa
;;
cmp.ne pt1 = 0, rpVa
(pt1) br.cond.spnt K4dtf10
;;
#endif
//
// deposit extra PFN bits for 4k page
//
dep rPte0 = rVa12, rPte0, PAGE4K_SHIFT, PAGE_SHIFT-PAGE4K_SHIFT // I
;;
(pClear)itc.d rPte0 // M, install PTE
;;
and rIndex = 7, rIndex // A
movl rpBuffer = KiPcr + PcForwardProgressBuffer // L
;;
shladd rpVa = rIndex, 4, rpBuffer // M
add rIndex = 1, rIndex // I
;;
st8 [rpVa] = rva // M
add rpPte = 8, rpVa // I
;;
st8 [rpPte] = rPte0 // M
//
// done updating the forward progress buffer entry, release spinlock
//
movl rpBuffer = KiPcr + PcFpbLock // L
;;
st8.rel [rpBuffer] = r0, PcForwardProgressBuffer - PcFpbLock
mov rOldIIP = rIA32IIP // I
#if !defined(NT_UP)
rps = h27
rAte2 = h28
rPte2 = h31
ld8.s rPte2 = [riha] // M
ld8.s rAte2 = [rpAte] // M
cmp.ne pPrg = zero, zero // I
;;
cmp.ne.or pPrg = rPte, rPte2 // M
tnat.nz.or pPrg = rPte2 // I
cmp.ne.or pPrg = rAte, rAte2 // M
tnat.nz.or pPrg = rAte2 // I
;;
(pPrg) ptc.l rva, rps // M
(pPrg) st8 [rpPte] = r0 // M
#endif
(pLoop) br.cond.spnt KiFillForwardProgressTb // B
K4dtf10:
mov pr = rpr, -1 // I
rfi;; // B
HANDLER_EXIT(Ki4KDataTlbFault)
//
// ++
//
// Routine:
//
// Ki4KInstTlbFault
//
// Description:
//
// Branched from KiInstTlbVector if PTE.Cache indicates the reserved
// encoding. Reads the corresponding ATE and creates a 4kb TB on the
// fly inserts it to the TLB.
//
// On entry:
//
// rva (h24) : offending virtual address
// riha(h25) : IHA address
// rpr (h26) : PDE contents
//
// Notes:
//
// --
HANDLER_ENTRY(Ki4KInstTlbFault)
rva = h24 // passed
riha = h25 // passed
rpr = h26 // passed
rps = h27
rPfn = h28
rpAte = h28
rAte = h29
rPte = h30
rAltBase = h31
rPte0 = h31
rAteMask = h22
rVa12 = h23
pBr = pt0
pIndr = pt1
pMiss = pt2
pPrg = pt3
pMiss2 = pt6
mov rps = PS_4K << PS_SHIFT // M
movl rAltBase = ALT4KB_BASE // L
shr.u rPfn = rva, PAGE4K_SHIFT // I
dep rAteMask = 1, r0, 32, 1 // Set rAteMask equal to 2**32
;;
ld8.s rPte = [riha] // M
cmp.ge pBr, pt7 = rva, rAteMask // M/I, Test to see if rva is out of bounds.
shladd rpAte = rPfn, PTE_SHIFT, rAltBase // I
;;
ld8.s rAte = [rpAte] // M
movl rAteMask = ATE_MASK // L
;;
tnat.nz pMiss = rPte // I
(pMiss) br.cond.spnt KiPageTableFault // B
(pt7) tnat.nz pMiss2 = rAte // I
(pMiss2)br.cond.spnt KiAltTableFault // B
tbit.z.or pBr = rPte, PTE_VALID // I
tbit.z.or pBr = rAte, PTE_VALID // I
tbit.z.or pBr = rAte, PTE_ACCESS // I
or rAteMask = rAte, rAteMask // M
tbit.nz pIndr, p0 = rAte, ATE_INDIRECT // I
(pBr) br.cond.spnt KiPageFault // B
dep rPte0 = 0, rPte, 2, 3 // I, make it WB
shr rVa12 = rpAte, PTE_SHIFT // I
(pIndr) br.cond.spnt KiPteIndirectFault // B
;;
mov cr.itir = rps // M
and rPte0 = rPte0, rAteMask // I
;;
//
// deposit extra PFN bits for 4k page
//
dep rPte0 = rVa12, rPte0, PAGE4K_SHIFT, PAGE_SHIFT-PAGE4K_SHIFT // I
;;
itc.i rPte0 // M, install PTE
;;
#if !defined(NT_UP)
rps = h27
rAte2 = h28
rPte2 = h31
ld8.s rPte2 = [riha] // M
ld8.s rAte2 = [rpAte] // M
cmp.ne pPrg = zero, zero // I
;;
cmp.ne.or pPrg = rPte, rPte2 // M
tnat.nz.or pPrg = rPte2 // I
cmp.ne.or pPrg = rAte, rAte2 // M
tnat.nz.or pPrg = rAte2 // I
;;
(pPrg) ptc.l rva, rps // M
#endif
mov pr = rpr, -1 // I
rfi;; // B
HANDLER_EXIT(Ki4KInstTlbFault)
//
// ++
//
// Routine:
//
// KiAltTableFault
//
// Description:
//
// Branched from Inst/DataAccessBitVector
// Inserts a missing PTE translation for the alt table.
//
// On entry:
//
// rva (h24) : offending virtual address
// riha (h25) : a offending PTE address
// rpr: (h26) : saved predicate
//
// Handle:
//
// --
HANDLER_ENTRY(KiAltTableFault)
rpAte = h28 // passed
rva = h24
riha = h25
rpr = h26 // passed
rPte = h27
rPte2 = h28
rps = h29
thash riha = rpAte // M
cmp.ne pt1 = r0, r0
mov rva = rpAte // I
;;
ld8.s rPte = [riha] // M
;;
tnat.nz pt0, p0 = rPte // I
tbit.z.or pt1, p0 = rPte, PTE_ACCESS
tbit.z.or pt1, p0 = rPte, PTE_VALID // I, if non-present page fault
(pt0) br.cond.spnt KiPageTableFault // B
(pt1) br.cond.spnt KiPteNotPresentFault // B
;;
mov cr.ifa = rva
;;
itc.d rPte // M
;;
#if !defined(NT_UP)
ld8.s rPte2 = [riha] // M
mov rps = PAGE_SHIFT << PS_SHIFT // I
cmp.ne pt0 = zero, zero // I
;;
cmp.ne.or pt0 = rPte2, rPte // M
tnat.nz.or pt0 = rPte2 // I
;;
(pt0) ptc.l rva, rps // M
#endif
mov pr = rpr, -1 // I
rfi;; // B
HANDLER_EXIT(KiAltTableFault)
//
// ++
//
// Routine:
//
// KiFillForwardProgressTb
//
// Description:
//
// Fill TB from TLB forward progress buffer.
//
// On entry:
//
// rpBuffer (h16) : forward progress buffer address
// rpr: (h26) : saved predicate
//
// Handle:
//
// --
HANDLER_ENTRY(KiFillForwardProgressTb)
rLc = h29
rT0 = h28
rps = h27
rpr = h26
rVa = h22
rPte = h21
rpVa = h19
rpPte = h17
rpBuffer= h16
mov rpVa = rpBuffer // A
mov.i rLc = ar.lc // I
mov rT0 = NUMBER_OF_FWP_ENTRIES - 1 //
;;
add rpPte = 8, rpBuffer // A
mov.i ar.lc = rT0 // I
;;
fpb_loop:
//
// use ALAT to see if somebody modify the PTE entry
//
ld8 rVa = [rpVa], 16 // M
ld8.a rPte = [rpPte] // M
;;
mov cr.ifa = rVa // M
cmp.ne pt0, pt1 = rPte, r0 // I
;;
(pt0) itc.d rPte // M
;;
(pt0) ld8.c.clr rPte = [rpPte] // M
add rpPte = 16, rpPte // I
;;
(pt1) invala.e rPte // M, invalidate ALAT entry
(pt0) cmp.eq.and pt0 = rPte, r0 // I
;;
(pt0) ptc.l rVa, rps // M
br.cloop.dptk.many fpb_loop;; // B
mov.i ar.lc = rLc
mov pr = rpr, -1 // I
rfi // B
;;
//
// Add four nop cycles (two bundles) bundles of NOP's to
// workaround McKinly Errata.
//
nop.m 0
;;
nop.m 0
;;
nop.m 0
;;
nop.m 0
;;
HANDLER_EXIT(KiFillForwardProgressTb)
//++
//
// Routine Description:
//
// This routine begins the common code for raising an exception.
// The routine saves the non-volatile state and dispatches to the
// next level exception dispatcher.
//
// Arguments:
//
// a0 - pointer to trap frame
// a1 - previous mode
//
// Return Value:
//
// None.
//
//--
NESTED_ENTRY(KiExceptionDispatch)
//
// Build exception frame
//
.regstk 2, 3, 5, 0
.prologue 0xA, loc0
alloc t16 = ar.pfs, 2, 3, 5, 0
mov loc0 = sp
cmp4.eq pt0 = UserMode, a1 // previous mode is user?
mov loc1 = brp
.fframe ExceptionFrameLength
add sp = -ExceptionFrameLength, sp
;;
.save ar.unat, loc2
mov loc2 = ar.unat
add t0 = ExFltS19+STACK_SCRATCH_AREA, sp
add t1 = ExFltS18+STACK_SCRATCH_AREA, sp
;;
.save.gf 0x0, 0xC0000
stf.spill [t0] = fs19, ExFltS17-ExFltS19
stf.spill [t1] = fs18, ExFltS16-ExFltS18
;;
.save.gf 0x0, 0x30000
stf.spill [t0] = fs17, ExFltS15-ExFltS17
stf.spill [t1] = fs16, ExFltS14-ExFltS16
mov t10 = bs4
;;
.save.gf 0x0, 0xC000
stf.spill [t0] = fs15, ExFltS13-ExFltS15
stf.spill [t1] = fs14, ExFltS12-ExFltS14
mov t11 = bs3
;;
.save.gf 0x0, 0x3000
stf.spill [t0] = fs13, ExFltS11-ExFltS13
stf.spill [t1] = fs12, ExFltS10-ExFltS12
mov t12 = bs2
;;
.save.gf 0x0, 0xC00
stf.spill [t0] = fs11, ExFltS9-ExFltS11
stf.spill [t1] = fs10, ExFltS8-ExFltS10
mov t13 = bs1
;;
.save.gf 0x0, 0x300
stf.spill [t0] = fs9, ExFltS7-ExFltS9
stf.spill [t1] = fs8, ExFltS6-ExFltS8
mov t14 = bs0
;;
.save.gf 0x0, 0xC0
stf.spill [t0] = fs7, ExFltS5-ExFltS7
stf.spill [t1] = fs6, ExFltS4-ExFltS6
mov t15 = ar.lc
;;
.save.gf 0x0, 0x30
stf.spill [t0] = fs5, ExFltS3-ExFltS5
stf.spill [t1] = fs4, ExFltS2-ExFltS4
;;
.save.f 0xC
stf.spill [t0] = fs3, ExFltS1-ExFltS3 // save fs3
stf.spill [t1] = fs2, ExFltS0-ExFltS2 // save fs2
;;
.save.f 0x3
stf.spill [t0] = fs1, ExBrS4-ExFltS1 // save fs1
stf.spill [t1] = fs0, ExBrS3-ExFltS0 // save fs0
;;
.save.b 0x18
st8 [t0] = t10, ExBrS2-ExBrS4 // save bs4
st8 [t1] = t11, ExBrS1-ExBrS3 // save bs3
;;
.save.b 0x6
st8 [t0] = t12, ExBrS0-ExBrS2 // save bs2
st8 [t1] = t13, ExIntS2-ExBrS1 // save bs1
;;
.save.b 0x1
st8 [t0] = t14, ExIntS3-ExBrS0 // save bs0
movl out0 = KiPcr+PcCurrentThread
;;
.save.gf 0xC, 0x0
.mem.offset 0,0
st8.spill [t0] = s3, ExIntS1-ExIntS3 // save s3
.mem.offset 8,0
st8.spill [t1] = s2, ExIntS0-ExIntS2 // save s2
;;
.save.gf 0x3, 0x0
.mem.offset 0,0
st8.spill [t0] = s1, ExApLC-ExIntS1 // save s1
.mem.offset 8,0
st8.spill [t1] = s0, ExApEC-ExIntS0 // save s0
;;
.savepsp ar.pfs, ExceptionFrameLength-ExApEC-STACK_SCRATCH_AREA
st8 [t1] = t16, ExIntNats-ExApEC
mov t4 = ar.unat // captured Nats of s0-s3
;;
(pt0) ld8 out0 = [out0]
;;
(pt0) add out0 = ThStackBase, out0
.savepsp ar.lc, ExceptionFrameLength-ExApLC-STACK_SCRATCH_AREA
st8 [t0] = t15
.savepsp @priunat, ExceptionFrameLength-ExIntNats-STACK_SCRATCH_AREA
st8 [t1] = t4 // save Nats of s0-s3
;;
PROLOGUE_END
(pt0) ld8 out0 = [out0]
;;
(pt0) add out0 = -ThreadStateSaveAreaLength+TsHigherFPVolatile, out0
(pt0) br.call.sptk brp = KiSaveHigherFPVolatile
;;
CAPSTART(KiExceptionDispatch,KiDispatchException)
add out0 = TrExceptionRecord, a0 // -> exception record
add out1 = STACK_SCRATCH_AREA, sp // -> exception frame
mov out2 = a0 // -> trap frame
mov out3 = a1 // previous mode
mov out4 = 1 // first chance
br.call.sptk.many brp = KiDispatchException
;;
CAPEND(KiExceptionDispatch)
add t1 = ExApEC+STACK_SCRATCH_AREA, sp
movl t0 = KiExceptionExit
;;
//
// Interrupts must be disabled before calling KiExceptionExit
// because the unwind code cannot unwind from that point.
//
FAST_DISABLE_INTERRUPTS
ld8 t1 = [t1]
mov brp = t0
;;
mov ar.unat = loc2
mov ar.pfs = t1
add s1 = STACK_SCRATCH_AREA, sp // s1 -> exception frame
mov s0 = a0 // s0 -> trap frame
br.ret.sptk brp
;;
NESTED_EXIT(KiExceptionDispatch)
//++
//
// BOOLEAN
// KeInvalidAccessAllowed (
// IN PVOID TrapInformation
// )
//
// Routine Description:
//
// Mm will pass a pointer to a trap frame prior to issuing a bug check on
// a pagefault. This routine lets Mm know if it is ok to bugcheck. The
// specific case we must protect are the interlocked pop sequences which can
// blindly access memory that may have been freed and/or reused prior to the
// access. We don't want to bugcheck the system in these cases, so we check
// the instruction pointer here.
//
// Arguments:
//
// TrapFrame (a0) - Supplies a trap frame pointer. NULL means return False.
//
// Return Value:
//
// True if the invalid access should be ignored.
// False which will usually trigger a bugcheck.
//
//--
LEAF_ENTRY(KeInvalidAccessAllowed)
.regstk 1, 0, 0, 0
cmp.eq pt0 = 0, a0
movl t1 = ExpInterlockedPopEntrySListFault
add t0 = TrStIIP, a0
mov v0 = zero // assume access not allowed
(pt0) br.ret.spnt brp
;;
ld8 t0 = [t0]
;;
cmp.eq pt2 = t0, t1
;;
nop.m 0
(pt2) mov v0 = 1
br.ret.sptk brp
LEAF_EXIT(KeInvalidAccessAllowed)
LEAF_ENTRY(KeScrubBackingStoreRegisters)
.regstk 1, 15, 1, 0
alloc loc0 = 1, 15, 1, 0
mov loc1 = brp
cmp.ge pt0, pt1 = 17, a0
add out0 = -16, a0
mov loc2 = r0
mov loc3 = r0
;;
mov loc4 = r0
mov loc5 = r0
mov loc6 = r0
mov loc7 = r0
mov loc8 = r0
mov loc9 = r0
mov loc10 = r0
mov loc11 = r0
mov loc12 = r0
mov loc13 = r0
mov loc14 = r0
(pt1) br.call.dpnt brp = KeScrubBackingStoreRegisters
mov out0 = r0
mov brp = loc1
mov loc1 = r0
mov a0 = r0
mov ar.pfs = loc0
mov loc0 = r0
br.ret.sptk brp
LEAF_EXIT (KeScrubBackingStoreRegisters)
LEAF_ENTRY(KiPageZeroFault)
rpr = h26
rK5 = h28
mov h29 = cr.ipsr
tbit.z pt1, pt0 = rK5, 0
;;
dep h29 = 1, h29, PSR_ED, 1 // I
(pt0) itc.d r0
;;
mov cr.ipsr = h29
(pt0) mov rK5 = r0
(pt1) add rK5 = 3, rK5
;;
mov ar.k5 = rK5
mov pr = rpr, -1 // I
rfi;; // B
LEAF_EXIT(KiPageZeroFault)
Reference in New Issue View Git Blame Copy Permalink