Microsoft/Windows2000
Microsoft/Windows2000
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Windows2000/private/ntos/ke/ppc/ctxswap.s
Microsoft 45f4a58588 First commit
2020-09-30 17:12:32 +02:00

2943 lines
98 KiB
ArmAsm
Raw Blame History

// TITLE("Context Swap")
// Copyright (c) 1994 IBM Corporation
// Module Name:
// ctxswap.s
// Abstract:
// This module implements the PowerPC machine dependent code necessary to
// field the dispatch interrupt and to perform kernel initiated context
// switching.
// Author:
// Peter L. Johnston (plj@vnet.ibm.com) August 1993
// Adapted from code by David N. Cutler (davec) 1-Apr-1991
// Environment:
// Kernel mode only.
// Revision History:
// plj Apr-94 Upgraded to NT 3.5.
#include "ksppc.h"
// Module Constants
#define rPrcb r.29
#define OTH r.30
#define NTH r.31
// Global externals
.extern ..KdPollBreakIn
.extern ..KeFlushCurrentTb
.extern ..KiActivateWaiterQueue
.extern ..KiContinueClientWait
.extern ..KiDeliverApc
.extern ..KiQuantumEnd
.extern ..KiReadyThread
.extern ..KiWaitTest
.extern KdDebuggerEnabled
.extern KeTickCount
.extern KiDispatcherReadyListHead
.extern KiIdleSummary
.extern KiReadySummary
.extern KiWaitInListHead
.extern KiWaitOutListHead
.extern __imp_HalProcessorIdle
.extern __imp_KeLowerIrql
#if DBG
.extern ..DbgBreakPoint
.extern ..DbgBreakPointWithStatus
#endif
#if !defined(NT_UP)
.extern KeNumberProcessors
.extern KiBarrierWait
.extern KiContextSwapLock
.extern KiDispatcherLock
.extern KiProcessorBlock
#if SPINDBG
.extern ..KiAcquireSpinLockDbg
.extern ..KiTryToAcquireSpinLockDbg
#endif
#endif
.extern KiMasterSequence
.extern KiMasterPid
.globl KiScheduleCount
.data
KiScheduleCount:
.long 0
#if COLLECT_PAGING_DATA
.globl KiFlushOnProcessSwap
.data
KiFlushOnProcessSwap:
.long 0
#endif
// ThreadSwitchFrame
// This is the layout of the beginning of the stack frame that must be
// established by any routine that calls SwapContext.
// The caller of SwapContext must have saved r.14 and 26 thru 31.
// SwapContext will take care of r.15 thru r.25, f.14 thru f.31 and
// the condition register.
// Note: this is not a complete stack frame, the caller must allocate
// additional space for any additional registers it needs to
// save (eg Link Register). (Also, the following has not been
// padded to 8 bytes).
// WARNING: KiInitializeContextThread() is aware of the layout of
// a ThreadSwitchFrame.
.struct 0
.space StackFrameHeaderLength
swFrame:.space SwapFrameLength
swFrameLength:
// SBTTL("Switch To Thread")
// NTSTATUS
// KiSwitchToThread (
// IN PKTHREAD NextThread,
// IN ULONG WaitReason,
// IN ULONG WaitMode,
// IN PKEVENT WaitObject
// )
// Routine Description:
// This function performs an optimal switch to the specified target thread
// if possible. No timeout is associated with the wait, thus the issuing
// thread will wait until the wait event is signaled or an APC is deliverd.
// N.B. This routine is called with the dispatcher database locked.
// N.B. The wait IRQL is assumed to be set for the current thread and the
// wait status is assumed to be set for the target thread.
// N.B. It is assumed that if a queue is associated with the target thread,
// then the concurrency count has been incremented.
// N.B. Control is returned from this function with the dispatcher database
// unlocked.
// Arguments:
// NextThread - Supplies a pointer to a dispatcher object of type thread.
// WaitReason - supplies the reason for the wait operation.
// WaitMode - Supplies the processor wait mode.
// WaitObject - Supplies a pointer to a dispatcher object of type event
// or semaphore.
// Return Value:
// The wait completion status. A value of STATUS_SUCCESS is returned if
// the specified object satisfied the wait. A value of STATUS_USER_APC is
// returned if the wait was aborted to deliver a user APC to the current
// thread.
.struct 0
.space swFrameLength
sttLR: .space 4
sttReason: .space 4
sttMode: .space 4
sttObject: .space 4
.align 3 // ensure 8 byte alignment
sttFrameLength:
SPECIAL_ENTRY_S(KiSwitchToThread,_TEXT$00)
mflr r.0 // get return address
stwu r.sp, -sttFrameLength(r.sp) // buy stack frame
stw r.14, swFrame + ExGpr14(r.sp) // save gpr 14
stw r.26, swFrame + ExGpr26(r.sp) // save gprs 26 through 31
stw r.27, swFrame + ExGpr27(r.sp)
stw r.28, swFrame + ExGpr28(r.sp)
stw r.29, swFrame + ExGpr29(r.sp)
stw r.30, swFrame + ExGpr30(r.sp)
stw r.31, swFrame + ExGpr31(r.sp)
ori NTH, r.3, 0
stw r.0, sttLR(r.sp) // save return address
li r.0, 0
PROLOGUE_END(KiSwitchToThread)
// Save the wait reason, the wait mode, and the wait object address.
stw r.4, sttReason(r.sp) // save wait reason
stw r.5, sttMode(r.sp) // save wait mode
stw r.6, sttObject(r.sp) // save wait object address
// If the target thread's kernel stack is resident, the target thread's
// process is in the balance set, the target thread can run on the
// current processor, and another thread has not already been selected
// to run on the current processor, then do a direct dispatch to the
// target thread bypassing all the general wait logic, thread priorities
// permiting.
lwz r.7, ThApcState + AsProcess(NTH) // get target process address
lbz r.8, ThKernelStackResident(NTH) // get kernel stack resident
lwz rPrcb, KiPcr + PcPrcb(r.0) // get address of PRCB
lbz r.10, PrState(r.7) // get target process state
lwz OTH, KiPcr + PcCurrentThread(r.0) // get current thread address
cmpwi r.8, 0 // kernel stack resident?
beq LongWay // if eq, kernel stack not resident
cmpwi r.10, ProcessInMemory // process in memory?
bne LongWay // if ne, process not in memory
#if !defined(NT_UP)
lwz r.8, PbNextThread(rPrcb) // get address of next thread
lbz r.10, ThNextProcessor(OTH) // get current processor number
lwz r.14, ThAffinity(NTH) // get target thread affinity
lwz r.26, KiPcr + PcSetMember(r.0) // get processor set member
cmpwi r.8, 0 // next thread selected?
bne LongWay // if ne, next thread selected
and. r.26, r.26, r.14 // check for compatible affinity
beq LongWay // if eq, affinity not compatible
#endif
// Compute the new thread priority.
lbz r.14, ThPriority(OTH) // get client thread priority
lbz r.26, ThPriority(NTH) // get server thread priority
cmpwi r.14, LOW_REALTIME_PRIORITY // check if realtime client
cmpwi cr.7, r.26, LOW_REALTIME_PRIORITY // check if realtime server
bge stt60 // if ge, realtime client
lbz r.27, ThPriorityDecrement(NTH) // get priority decrement value
lbz r.28, ThBasePriority(NTH) // get server base priority
bge cr.7, stt50 // if ge, realtime server
addi r.9, r.28, 1 // computed boosted priority
cmpwi r.27, 0 // server boot active?
bne stt30 // if ne, server boost active
// Both the client and the server are not realtime and a priority boost
// is not currently active for the server. Under these conditions an
// optimal switch to the server can be performed if the base priority
// of the server is above a minimum threshold or the boosted priority
// of the server is not less than the client priority.
cmpw r.9, r.14 // check if high enough boost
cmpwi cr.7, r.9, LOW_REALTIME_PRIORITY // check if less than realtime
blt stt20 // if lt, boosted priority less
stb r.9, ThPriority(NTH) // asssume boosted priority is okay
blt cr.7, stt70 // if lt, less than realtime
li r.9, LOW_REALTIME_PRIORITY - 1 // set high server priority
stb r.9, ThPriority(NTH) //
b stt70
stt20:
// The boosted priority of the server is less than the current priority of
// the client. If the server base priority is above the required threshold,
// then a optimal switch to the server can be performed by temporarily
// raising the priority of the server to that of the client.
cmpwi r.28, BASE_PRIORITY_THRESHOLD // check if above threshold
sub r.9, r.14, r.28 // compute priority decrement value
blt LongWay // if lt, priority below threshold
li r.28, ROUND_TRIP_DECREMENT_COUNT // get system decrement count value
stb r.9, ThPriorityDecrement(NTH) // set priority decrement value
stb r.14, ThPriority(NTH) // set current server priority
stb r.28, ThDecrementCount(NTH) // set server decrement count
b stt70
stt30:
// A server boost has previously been applied to the server thread. Count
// down the decrement count to determine if another optimal server switch
// is allowed.
lbz r.9, ThDecrementCount(NTH) // decrement server count value
subic. r.9, r.9, 1 //
stb r.9, ThDecrementCount(NTH) // store updated decrement count
beq stt40 // if eq, no more switches allowed
// Another optimal switch to the server is allowed provided that the
// server priority is not less than the client priority.
cmpw r.26, r.14 // check if server lower priority
bge stt70 // if ge, server not lower priority
b LongWay
stt40:
// The server has exhausted the number of times an optimal switch may
// be performed without reducing its priority. Reduce the priority of
// the server to its original unboosted value minus one.
stb r.0, ThPriorityDecrement(NTH) // clear server priority decrement
stb r.28, ThPriority(NTH) // set server priority to base
b LongWay
stt50:
// The client is not realtime and the server is realtime. An optimal switch
// to the server can be performed.
lbz r.9, PrThreadQuantum(r.7) // get process quantum value
b stt65
stt60:
// The client is realtime. In order for an optimal switch to occur, the
// server must also be realtime and run at a high or equal priority.
cmpw r.26, r.14 // check if server is lower priority
lbz r.9, PrThreadQuantum(r.7) // get process quantum value
blt LongWay // if lt, server is lower priority
stt65:
stb r.9, ThQuantum(NTH) // set server thread quantum
stt70:
// Set the next processor for the server thread.
#if !defined(NT_UP)
stb r.10, ThNextProcessor(NTH) // set server next processor number
#endif
// Set the address of the wait block list in the client thread, initialize
// the event wait block, and insert the wait block in client event wait list.
addi r.8, OTH, EVENT_WAIT_BLOCK_OFFSET // compute wait block address
stw r.8, ThWaitBlockList(OTH) // set address of wait block list
stw r.0, ThWaitStatus(OTH) // set initial wait status
stw r.6, WbObject(r.8) // set address of wait object
stw r.8, WbNextWaitBlock(r.8) // set next wait block address
lis r.10, WaitAny // get wait type and wait key
stw r.10, WbWaitKey(r.8) // set wait key and wait type
addi r.10, r.6, EvWaitListHead // compute wait object listhead address
lwz r.14, LsBlink(r.10) // get backward link of listhead
addi r.26, r.8, WbWaitListEntry // compute wait block list entry address
stw r.26, LsBlink(r.10) // set backward link of listhead
stw r.26, LsFlink(r.14) // set forward link in last entry
stw r.10, LsFlink(r.26) // set forward link in wait entry
stw r.14, LsBlink(r.26) // set backward link in wait entry
// Set the client thread wait parameters, set the thread state to Waiting,
// and insert the thread in the proper wait list.
stb r.0, ThAlertable(OTH) // set alertable FALSE.
stb r.4, ThWaitReason(OTH) // set wait reason
stb r.5, ThWaitMode(OTH) // set the wait mode
lbz r.6, ThEnableStackSwap(OTH) // get kernel stack swap enable
lwz r.10, [toc]KeTickCount(r.toc) // get &KeTickCount
lwz r.10, 0(r.10) // get low part of tick count
stw r.10, ThWaitTime(OTH) // set thread wait time
li r.8, Waiting // set thread state
stb r.8, ThState(OTH) //
lwz r.8,[toc]KiWaitInListHead(r.toc) // get address of wait in listhead
cmpwi r.5, 0 // is wait mode kernel?
beq stt75 // if eq, wait mode is kernel
cmpwi r.6, 0 // is kernel stack swap disabled?
beq stt75 // if eq, kernel stack swap disabled
cmpwi r.14, LOW_REALTIME_PRIORITY + 9 // check if priority in range
blt stt76 // if lt, thread priority in range
stt75:
lwz r.8,[toc]KiWaitOutListHead(r.toc) // get address of wait in listhead
stt76:
lwz r.14, LsBlink(r.8) // get backlink of wait listhead
addi r.26, OTH, ThWaitListEntry // compute wait list entry address
stw r.26, LsBlink(r.8) // set backward link of listhead
stw r.26, LsFlink(r.14) // set forward link in last entry
stw r.8, LsFlink(r.26) // set forward link in wait entry
stw r.14, LsBlink(r.26) // set backward link in wait entry
stt77:
// If the current thread is processing a queue entry, then attempt to
// activate another thread that is blocked on the queue object.
// N.B. The next thread address can change if the routine to activate
// a queue waiter is called.
lwz r.3, ThQueue(OTH) // get queue object address
cmpwi r.3, 0 // queue object attached?
beq stt78 // if eq, no queue object attached
stw NTH, PbNextThread(rPrcb) // set next thread address
bl ..KiActivateWaiterQueue // attempt to activate a blocked thread
lwz NTH, PbNextThread(rPrcb) // get next thread address
li r.0, 0
stw r.0, PbNextThread(rPrcb) // set next thread address to NULL
stt78:
#if !defined(NT_UP)
lwz r.27, [toc]KiContextSwapLock(r.2)// get &KiContextSwapLock
lwz r.28, [toc]KiDispatcherLock(r.2) // get &KiDispatcherLock
#endif
stw NTH, PbCurrentThread(rPrcb) // set address of current thread object
bl ..SwapContext // swap context
// Lower IRQL to its previous level.
// N.B. SwapContext releases the dispatcher database lock.
// N.B. Register NTH (r.31) contains the address of the new thread on return.
// In the following, we could lower IRQL, isync then check for pending
// interrupts. I believe it is faster to disable interrupts and get
// both loads going. We need to avoid the situation where a DPC or
// APC could be queued between the time we load PcSoftwareInterrupt
// and actually lowering IRQL. (plj)
// We load the thread's WaitStatus in this block for scheduling
// reasons in the hope that the normal case will be there is NOT
// a DPC or APC pending.
lbz r.27, ThWaitIrql(NTH) // get original IRQL
DISABLE_INTERRUPTS(r.5, r.6)
lhz r.4, KiPcr+PcSoftwareInterrupt(r.0)
lwz r.26, ThWaitStatus(NTH) // get wait completion status
stb r.27, KiPcr+PcCurrentIrql(r.0) // set new IRQL
ENABLE_INTERRUPTS(r.5)
cmpw r.4, r.27 // check if new IRQL allows
ble+ stt79 // APC/DPC and one is pending
bl ..KiDispatchSoftwareInterrupt // process software interrupt
stt79:
// If the wait was not interrupted to deliver a kernel APC, then return the
// completion status.
cmpwi r.26, STATUS_KERNEL_APC // check if awakened for kernel APC
ori r.3, r.26, 0 // set return status
bne stt90 // if ne, normal wait completion
// Disable interrupts and acquire the dispatcher database lock.
#if !defined(NT_UP)
DISABLE_INTERRUPTS(r.5, r.6)
// WARNING: The address of KiDispatcherLock was intentionally left in
// r.28 by SwapContext for use here.
ACQUIRE_SPIN_LOCK(r.28, NTH, r.7, stt80, stt82)
#endif
// Raise IRQL to synchronization level and save wait IRQL.
li r.7, SYNCH_LEVEL
stb r.7, KiPcr+PcCurrentIrql(r.0)
#if !defined(NT_UP)
ENABLE_INTERRUPTS(r.5)
#endif
stb r.27, ThWaitIrql(NTH) // set client wait IRQL
b ContinueWait
#if !defined(NT_UP)
SPIN_ON_SPIN_LOCK_ENABLED(r.28, r.7, stt80, stt82, stt85, r.5, r.6)
#endif
LongWay:
// Ready the target thread for execution and wait on the specified wait
// object.
bl ..KiReadyThread // ready thread for execution
// Continue the wait and return the wait completion status.
// N.B. The wait continuation routine is called with the dispatcher
// database locked.
ContinueWait:
lwz r.3, sttObject(r.sp) // get wait object address
lwz r.4, sttReason(r.sp) // get wait reason
lwz r.5, sttMode(r.sp) // get wait mode
bl ..KiContinueClientWait // continue client wait
stt90:
lwz r.0, sttLR(r.sp) // restore return address
lwz r.26, swFrame + ExGpr26(r.sp) // restore gprs 26 thru 31
lwz r.27, swFrame + ExGpr27(r.sp) //
lwz r.28, swFrame + ExGpr28(r.sp) //
lwz r.29, swFrame + ExGpr29(r.sp) //
lwz r.30, swFrame + ExGpr30(r.sp) //
mtlr r.0 // set return address
lwz r.31, swFrame + ExGpr31(r.sp) //
lwz r.14, swFrame + ExGpr14(r.sp) // restore gpr 14
addi r.sp, r.sp, sttFrameLength // return stack frame
blr
DUMMY_EXIT(KiSwitchToThread)
SBTTL("Dispatch Software Interrupt")
// VOID
// KiDispatchSoftwareInterrupt (VOID)
// Routine Description:
// This routine is called when the current irql drops below
// DISPATCH_LEVEL and a software interrupt may be pending. A
// software interrupt is either a pending DPC or a pending APC.
// If a DPC is pending, Irql is raised to DISPATCH_LEVEL and
// KiDispatchInterrupt is called. When KiDispatchInterrupt
// returns, the Irql (before we raised it) is compared to
// APC_LEVEL and if less and an APC interrupt is pending it
// is processed.
// Note: this routine manipulates PCR->CurrentIrql directly to
// avoid recursion by using Ke[Raise|Lower]Irql.
// Arguments:
// None.
// Return Value:
// None.
.text
.align 5 // cache block align
LEAF_ENTRY_S(KiDispatchSoftwareInterrupt, _TEXT$00)
li r.3, 1 // Flag to dispatch routines
// to enable interrupts when
// returning to IRQL 0
DISABLE_INTERRUPTS(r.7, r.4)
ALTERNATE_ENTRY(KiDispatchSoftwareIntDisabled)
stw r.3, -8(r.sp) // Flag to dispatch routines
// indicating whether to enable
// or not to enable interrupts
// when returning to IRQL 0
lhz r.9, KiPcr+PcSoftwareInterrupt(r.0) // pending s/w interrupt?
lbz r.3, KiPcr+PcCurrentIrql(r.0) // get current irql
srwi. r.4, r.9, 8 // isolate DPC pending
cmpw cr.6, r.9, r.3 // see if APC int and APC level
cmpwi cr.7, r.3, APC_LEVEL // compare IRQL to APC LEVEL
// Possible values for SoftwareInterrupt (r.9) are
// 0x0101 DPC and APC interrupt pending
// 0x0100 DPC interrupt pending
// 0x0001 APC interrupt pending
// 0x0000 No software interrupt pending (unlikely but possible)
// Possible values for current IRQL are zero or one. By comparing
// SoftwareInterrupt against current IQRL (above) we can quickly see
// if any software interrupts are valid at this time.
// Calculate correct IRQL for the interrupt we are processing. If DPC
// then we need to be at DISPATCH_LEVEL which is one greater than APC_
// LEVEL. r.4 contains one if we are going to run a DPC, so we add
// APC_LEVEL to r.4 to get the desired IRQL.
addi r.4, r.4, APC_LEVEL // calculate new IRQL
ble cr.6,ExitEnabled // return if no valid interrupt
#if DBG
cmplwi cr.6, r.3, DISPATCH_LEVEL // sanity check, should only
blt+ cr.6, $+12 // below DISPATCH_LEVEL
twi 31, 0, 0x16 // BREAKPOINT
blr // return if wrong IRQL
#endif
// ..DispatchSoftwareInterrupt is an alternate entry used indirectly
// by KeReleaseSpinLock (via KxReleaseSpinLock). KeReleaseSpinLock has
// been carefully written to construct the same conditions as apply if
// execution came from above.
..DispatchSoftwareInterrupt:
stb r.4, KiPcr+PcCurrentIrql(r.0) // set IRQL
ENABLE_INTERRUPTS(r.7)
beq cr.0, ..KiDispatchApc // jif not DPC interrupt
beq cr.7, ..KiDispatchDpcOnly // jif DPC and old IRQL APC LEV.
b ..KiDispatchDpc // DPC int, old IRQL < APC LEV.
ExitEnabled:
ENABLE_INTERRUPTS(r.7)
blr
DUMMY_EXIT(KiDispatchSoftwareInterrupt)
SBTTL("Unlock Dispatcher Database")
// VOID
// KiUnlockDispatcherDatabase (
// IN KIRQL OldIrql
// )
// Routine Description:
// This routine is entered at synchronization level with the dispatcher
// database locked. Its function is to either unlock the dispatcher
// database and return or initiate a context switch if another thread
// has been selected for execution.
// N.B. This code merges with the following swap context code.
// N.B. A context switch CANNOT be initiated if the previous IRQL
// is greater than or equal to DISPATCH_LEVEL.
// N.B. This routine is carefully written to be a leaf function. If,
// however, a context swap should be performed, the routine is
// switched to a nested function.
// Arguments:
// OldIrql (r.3) - Supplies the IRQL when the dispatcher database
// lock was acquired.
// Return Value:
// None.
LEAF_ENTRY_S(KiUnlockDispatcherDatabase, _TEXT$00)
// Check if a thread has been scheduled to execute on the current processor.
cmpwi cr.1, r.3, APC_LEVEL // check if IRQL below dispatch level
lwz r.7, KiPcr+PcPrcb(r.0) // get address of PRCB
lhz r.9, KiPcr+PcSoftwareInterrupt(r.0) // pending s/w interrupt?
lwz r.8, PbNextThread(r.7) // get next thread address
cmpw cr.7, r.9, r.3 // compare pending against irql
cmpwi r.8, 0 // check if next thread selected
bne uddSwitch // jif new thread selected
// Not switching, release dispatcher database lock.
#if !defined(NT_UP)
lwz r.5, [toc]KiDispatcherLock(r.toc)
li r.6, 0
RELEASE_SPIN_LOCK(r.5, r.6)
#endif
// If already at dispatch level, we're done.
bgtlr cr.1 // return
// Dropping below DISPATCH_LEVEL, we may need to inspire a software
// interrupt if one is pending.
// Above we compared r.9 (SoftwareInterrupt) with r.3 (OldIrql) (result in
// cr.7). See KiDispatchSoftwareInterrupt (above) for possible values.
// If we did not take the above branch then OldIrql is either zero or one.
// If SoftwareInterrupt is greater than OldIrql then a pending software
// interrupt can be taken at this time.
bgt cr.7, uddIntPending // jif pending interrupt
stb r.3, KiPcr+PcCurrentIrql(r.0) // set new IRQL
blr // return
// A new thread has been selected to run on the current processor, but
// the new IRQL is not below dispatch level. Release the dispatcher lock.
// If the current processor is not executing a DPC, then request a dispatch
// interrupt on the current. IRQL is already at the right level.
uddCantSwitch:
#if !defined(NT_UP)
lwz r.5, [toc]KiDispatcherLock(r.toc)
li r.6, 0
RELEASE_SPIN_LOCK(r.5, r.6)
#endif
lwz r.6, PbDpcRoutineActive(r.7)
li r.9, 1
cmplwi r.6, 0
bnelr // return if DPC already active
lwz r.5, [toc]KiScheduleCount(r.toc)
stb r.9, KiPcr+PcDispatchInterrupt(r.0) // request dispatch interrupt
lwz r.6, 0(r.5)
addi r.6, r.6, 1 // bump schedule count
stw r.6, 0(r.5)
blr // return
// A software interrupt is pending with higher priority than OldIrql.
// cr.1 is the result of comparing OldIrql with APC_LEVEL.
uddIntPending:
// Set flag to enable interrupts after dispatching software interrupts.
// r.9 must be non-zero because PcSoftwareInt(r.9) > OldIrql. Only
// needs to be set once no matter which dispatch routine is called.
stw r.9, -8(r.sp)
beq cr.1, ..KiDispatchDpcOnly // new IRQL doesn't allow APCs
srwi. r.3, r.9, 8 // isolate DPC pending
addi r.3, r.3, APC_LEVEL // calculate correct IRQL
stb r.3, KiPcr+PcCurrentIrql(r.0) // set new IRQL
bne ..KiDispatchDpc // jif DPC at APC_LEVEL
// IRQL dropped from DISPATCH_LEVEL to APC_LEVEL, make sure no DPCs
// were queued while we were checking. We are now at APC level so
// any new DPCs will happen without our having to check again.
lbz r.4, KiPcr+PcDispatchInterrupt(r.0)
cmpwi r.4, 0 // new DPCs?
beq ..KiDispatchApc // jif not
li r.3, DISPATCH_LEVEL // re-raise to DISPATCH_LEVEL
stb r.3, KiPcr+PcCurrentIrql(r.0) // set new IRQL
b ..KiDispatchDpc
DUMMY_EXIT(KiUnlockDispatcherDatabase)
// A new thread has been selected to run on the current processor.
// If the new IRQL is less than dispatch level, then switch to the new
// thread.
uddSwitch:
bgt cr.1, uddCantSwitch // jif new IRQL > apc level
// N.B. This routine is carefully written as a nested function. Control
// drops into this function from above.
SPECIAL_ENTRY_S(KxUnlockDispatcherDatabase, _TEXT$00)
mflr r.0 // get return address
stwu r.sp, -kscFrameLength(r.sp) // buy stack frame
stw r.29, swFrame + ExGpr29(r.sp) // save gpr 29
stw r.30, swFrame + ExGpr30(r.sp) // save gpr 30
ori rPrcb, r.7, 0 // copy PRCB address
stw r.31, swFrame + ExGpr31(r.sp) // save gpr 31
lwz OTH, KiPcr+PcCurrentThread(r.0) // get current thread address
stw r.27, swFrame + ExGpr27(r.sp) // save gpr 27
ori NTH, r.8, 0 // thread to switch to
stw r.14, swFrame + ExGpr14(r.sp) // save gpr 14
li r.11,0
stw r.28, swFrame + ExGpr28(r.sp) // save gpr 28
stw r.26, swFrame + ExGpr26(r.sp) // save gpr 26
stw r.0, kscLR(r.sp) // save return address
PROLOGUE_END(KxUnlockDispatcherDatabase)
stw r.11, PbNextThread(rPrcb) // clear next thread address
stb r.3, ThWaitIrql(OTH) // save previous IRQL
// Reready current thread for execution and swap context to the selected thread.
ori r.3, OTH, 0 // set address of previous thread object
stw NTH, PbCurrentThread(rPrcb) // set address of current thread object
#if !defined(NT_UP)
lwz r.27,[toc]KiContextSwapLock(r.2)// get &KiContextSwapLock
lwz r.28,[toc]KiDispatcherLock(r.2) // get &KiDispatcherLock
#endif
bl ..KiReadyThread // reready thread for execution
b ksc130 // join common code
DUMMY_EXIT(KxUnlockDispatcherDatabase)
// VOID
// KxReleaseSpinLock(VOID)
// Routine Description:
// This routine is entered when a call to KeReleaseSpinLock lowers
// IRQL to a level sufficiently low for a pending software interrupt
// to be deliverable.
// Although this routine has no arguments, the following entry conditions
// apply.
// Interrupts are disabled.
// r.7 MSR prior to disabling interrupts.
// r.4 IRQL to be raised to (PCR->CurrentIrql has been lowered
// even though interrupts are currently disabled).
// cr.0 ne if DPC pending
// cr.7 eq if ONLY DPCs can run (ie PCR->CurrentIrql == APC_LEVEL)
// Arguments:
// None.
// Return Value:
// None.
.struct 0
.space StackFrameHeaderLength
sp31: .space 4 // r.31 save
.align 3 // ensure 8 byte alignment
spLength:
SPECIAL_ENTRY_S(KxReleaseSpinLock, _TEXT$01)
stw r.31, sp31-spLength(r.sp) // save r.31
mflr r.31 // save return address (in r.31)
stwu r.sp, -spLength(r.sp) // buy stack frame
PROLOGUE_END(KxReleaseSpinLock)
li r.3, 1
stw r.3, -8(r.sp) // flag to dispatch routines
// to enable interrupts when
// returning to irql 0.
bl ..DispatchSoftwareInterrupt
mtlr r.31 // set return address
lwz r.31, sp31(r.sp) // restore r.31
addi r.sp, r.sp, spLength // release stack frame
SPECIAL_EXIT(KxReleaseSpinLock)
// VOID
// KiDispatchDpcOnly (VOID)
// Routine Description:
// This routine is entered as a result of lowering IRQL to
// APC_LEVEL with a DPC interrupt pending. IRQL is currently
// at DISPATCH_LEVEL, the dispatcher database is unlocked.
// Arguments:
// None.
// Return Value:
// None.
.set kdsiFrameLength, STK_MIN_FRAME+8
NESTED_ENTRY_S(KiDispatchDpcOnly, kdsiFrameLength, 0, 0, _TEXT$00)
PROLOGUE_END(KiDispatchDpcOnly)
kddo10: bl ..KiDispatchInterrupt
li r.3, APC_LEVEL // get new IRQL
DISABLE_INTERRUPTS(r.8, r.4)
lbz r.4, KiPcr+PcDispatchInterrupt(r.0) // more DPCs pending?
lwz r.5, KiPcr+PcPrcb(r.0) // get address of PRCB
cmpwi r.4, 0
lwz r.6, PbInterruptCount(r.5) // bump interrupt count
addi r.4, r.4, APC_LEVEL // calc new IRQL
addi r.6, r.6, 1
stw r.6, PbInterruptCount(r.5)
lwz r.6, STK_MIN_FRAME(r.sp) // parameter to enable
stb r.4, KiPcr+PcCurrentIrql(r.0) // set new IRQL
beq+ kddo20
ENABLE_INTERRUPTS(r.8)
b kddo10 // jif more DPCs to run
kddo20: cmpwi r.6, 0 // o.k to enable interrupts?
beq kddo25 // return if not
ENABLE_INTERRUPTS(r.8) // reenable interrupts and exit
kddo25:
NESTED_EXIT(KiDispatchDpcOnly, kdsiFrameLength, 0, 0)
// VOID
// KiDispatchDpc (VOID)
// Routine Description:
// This routine is entered as a result of lowering IRQL below
// APC_LEVEL with a DPC interrupt pending. IRQL is currently
// at DISPATCH_LEVEL, the dispatcher database is unlocked.
// Once DPC processing is complete, APC processing may be required.
// Arguments:
// None.
// Return Value:
// None.
NESTED_ENTRY_S(KiDispatchDpc, kdsiFrameLength, 0, 0, _TEXT$00)
PROLOGUE_END(KiDispatchDpc)
kdd10: bl ..KiDispatchInterrupt
DISABLE_INTERRUPTS(r.8, r.4)
lwz r.5, KiPcr+PcPrcb(r.0) // get address of PRCB
lhz r.4, KiPcr+PcSoftwareInterrupt(r.0) // more DPCs or APCs pending?
lwz r.6, PbInterruptCount(r.5) // bump interrupt count
cmpwi r.4, APC_LEVEL
addi r.6, r.6, 1
stw r.6, PbInterruptCount(r.5)
bgt- kdd20 // jif more DPCs
lwz r.6, STK_MIN_FRAME(r.sp) // parameter to enable
// interrupts
stb r.4, KiPcr+PcCurrentIrql(r.0) // set new IRQL
blt kdd30 // honor parameter to exit
// enabled or disabled when
// returning to IRQL 0
ENABLE_INTERRUPTS(r.8) // equal
b kda10 // jif APCs to run
kdd20: ENABLE_INTERRUPTS(r.8) // greater than
b kdd10 // jif more DPCs to run
kdd30: cmpwi r.6, 0 // o.k to enable interrupts?
beq- kdd35 // return if not
ENABLE_INTERRUPTS(r.8) // reenable interrupts and exit
kdd35:
NESTED_EXIT(KiDispatchDpc, kdsiFrameLength, 0, 0)
// VOID
// KiDispatchApc (VOID)
// Routine Description:
// This routine is entered as a result of lowering IRQL below
// APC_LEVEL with an APC interrupt pending. IRQL is currently
// at APC_LEVEL.
// Arguments:
// None.
// Return Value:
// None.
NESTED_ENTRY_S(KiDispatchApc, kdsiFrameLength, 0, 0, _TEXT$00)
PROLOGUE_END(KiDispatchApc)
kda10:
lwz r.6, KiPcr+PcPrcb(r.0) // get address of PRCB
li r.3, 0 // PreviousMode = Kernel
lwz r.7, PbApcBypassCount(r.6) // get APC bypass count
li r.4, 0 // TrapFrame = 0
li r.5, 0 // ExceptionFrame = 0
addi r.7, r.7, 1 // increment APC bypass count
stb r.3, KiPcr+PcApcInterrupt(r.0) // clear APC pending
stw r.7, PbApcBypassCount(r.6) // store new APC bypass count
bl ..KiDeliverApc
li r.3, 0 // get new IRQL
#if !defined(NT_UP)
DISABLE_INTERRUPTS(r.8, r.4)
#else
stb r.3, KiPcr+PcCurrentIrql(r.0) // lower IRQL
sync
#endif
lwz r.5, KiPcr+PcPrcb(r.0) // get address of PRCB
lbz r.4, KiPcr+PcApcInterrupt(r.0) // more APCs pending?
lwz r.6, PbInterruptCount(r.5) // bump interrupt count
cmpwi r.4, 0
addi r.6, r.6, 1
stw r.6, PbInterruptCount(r.5)
stb r.4, KiPcr+PcCurrentIrql(r.0) // Raise IRQL if more APCs
#if !defined(NT_UP)
ENABLE_INTERRUPTS(r.8)
#endif
bne- kda10 // jif more APCs to run
NESTED_EXIT(KiDispatchApc, kdsiFrameLength, 0, 0)
SBTTL("Swap Thread")
// VOID
// KiSwapThread (
// VOID
// )
// Routine Description:
// This routine is called to select the next thread to run on the
// current processor and to perform a context switch to the thread.
// Arguments:
// None.
// Outputs: ( for call to SwapContext )
// r.31 NTH pointer to new thread
// r.30 OTH pointer to old thread
// r.29 rPrcb pointer to processor control block
// r.28 pointer to dispatcher database lock
// r.27 pointer to context swap lock
// Return Value:
// Wait completion status (r.3).
.struct 0
.space swFrameLength
kscLR: .space 4
.align 3 // ensure 8 byte alignment
kscFrameLength:
.align 6 // cache line align
SPECIAL_ENTRY_S(KiSwapThread,_TEXT$00)
mflr r.0 // get return address
stwu r.sp, -kscFrameLength(r.sp) // buy stack frame
stw r.29, swFrame + ExGpr29(r.sp) // save gpr 29
stw r.30, swFrame + ExGpr30(r.sp) // save gpr 30
stw r.31, swFrame + ExGpr31(r.sp) // save gpr 31
lwz OTH, KiPcr+PcCurrentThread(r.0) // get current thread addr
lwz rPrcb, KiPcr+PcPrcb(r.0) // get Processor Control Block
stw r.27, swFrame + ExGpr27(r.sp) // save gpr 27
stw r.28, swFrame + ExGpr28(r.sp) // save gpr 28
stw r.14, swFrame + ExGpr14(r.sp) // save gpr 14
lwz r.27, [toc]KiReadySummary(r.toc) // get &KiReadySummary
lwz NTH, PbNextThread(rPrcb) // get address of next thread
stw r.26, swFrame + ExGpr26(r.sp) // save gpr 26
li r.28, 0 // load a 0
lwz r.26, 0(r.27) // get ready summary
cmpwi NTH, 0 // next thread selected?
stw r.0, kscLR(r.sp) // save return address
PROLOGUE_END(KiSwapThread)
stw r.28, PbNextThread(rPrcb) // zero address of next thread
bne ksc120 // if ne, next thread selected
#if !defined(NT_UP)
lwz r.5, [toc]KeTickCount(r.toc) // get &KeTickCount
lwz r.3, KiPcr+PcSetMember(r.0) // get processor affinity mask
lbz r.4, PbNumber(rPrcb) // get current processor number
lwz r.5, 0(r.5) // get low part of tick count
#endif
// Find the highest priority ready thread.
cntlzw r.6, r.26 // count zero bits from left
lwz r.8, [toc]KiDispatcherReadyListHead(r.toc) // get ready listhead base address
slw. r.7, r.26, r.6 // shift first set bit into sign bit
subfic r.6, r.6, 31 // convert shift count to priority
beq kscIdle // if mask is zero, no ready threads
kscReadyScan:
// If the thread can execute on the current processor, then remove it from
// the dispatcher ready queue.
slwi r.9, r.6, 3 // compute ready listhead offset
slwi r.7, r.7, 1 // position next ready summary bit
add r.9, r.9, r.8 // compute ready queue address
lwz r.10, LsFlink(r.9) // get address of first entry
subi NTH, r.10, ThWaitListEntry // compute address of thread object
#if !defined(NT_UP)
kscAffinityScan:
lwz r.11, ThAffinity(NTH) // get thread affinity
lbz r.0, ThNextProcessor(NTH) // get last processor number
and. r.11, r.11, r.3 // check for compatible thread affinity
cmpw cr.6, r.0, r.4 // compare last processor with current
bne kscAffinityOk // if ne, thread affinity compatible
lwz r.10, LsFlink(r.10) // get address of next entry
cmpw r.10, r.9 // compare with queue address
subi NTH, r.10, ThWaitListEntry // compute address of thread object
bne kscAffinityScan // if ne, not end of list
ksc70:
cmpwi r.7, 0 // more ready queues to scan?
subi r.6, r.6, 1 // decrement ready queue priority
blt kscReadyScan // if lt, queue contains an entry
beq kscIdle // if eq, no ready threads
slwi r.7, r.7, 1 // position next ready summary bit
b ksc70 // check next bit
kscAffinityOk:
// If the thread last ran on the current processor, has been waiting for
// longer than a quantum, or its priority is greater than low realtime
// plus 9, then select the thread. Otherwise, an attempt is made to find
// a more appropriate candidate.
beq cr.6, kscReadyFound // if eq, processor number match
lbz r.0, ThIdealProcessor(NTH) // get ideal processor number
cmpwi r.6, LOW_REALTIME_PRIORITY + 9 // check if priority in range
cmpw cr.6, r.0, r.4 // compare ideal processor with current
beq cr.6, ksc100 // if eq, processor number match
bge ksc100 // if ge, priority high enough
lwz r.12, ThWaitTime(NTH) // get time of thread ready
sub r.12, r.5, r.12 // compute length of wait
cmpwi cr.7, r.12, READY_SKIP_QUANTUM + 1 // check if wait time exceeded
bge cr.7, ksc100 // if ge, waited long enough
// Search forward in the ready queue until the end of the list is reached
// or a more appropriate thread is found.
lwz r.14, LsFlink(r.10) // get address of next entry
ksc80:
cmpw r.14, r.9 // compare with queue address
subi r.28, r.14, ThWaitListEntry // compute address of thread object
beq ksc100 // if eq, end of list
lwz r.11, ThAffinity(r.28) // get thread affinity
lbz r.0, ThNextProcessor(r.28) // get last processor number
and. r.11, r.11, r.3 // check for compatible thread affinity
cmpw cr.6, r.0, r.4 // compare last processor with current
beq ksc85 // if eq, thread affinity not compatible
beq cr.6, ksc90 // if eq, processor number match
lbz r.0, ThIdealProcessor(NTH) // get ideal processor number
cmpw cr.6, r.0, r.4 // compare ideal processor with current
beq cr.6, ksc90 // if eq, processor number match
ksc85:
lwz r.12, ThWaitTime(r.28) // get time of thread ready
lwz r.14, LsFlink(r.14) // get address of next entry
sub r.12, r.5, r.12 // compute length of wait
cmpwi cr.7, r.12, READY_SKIP_QUANTUM + 1 // check if wait time exceeded
blt cr.7, ksc80 // if lt, wait time not exceeded
b ksc100 // wait time exceeded -- switch to
// first matching thread in ready queue
ksc90:
ori NTH, r.28, 0 // set thread address
ori r.10, r.14, 0 // set list entry address
ksc100:
stb r.4, ThNextProcessor(NTH) // set next processor number
kscReadyFound:
#endif
// Remove the selected thread from the ready queue.
lwz r.11, LsFlink(r.10) // get list entry forward link
lwz r.12, LsBlink(r.10) // get list entry backward link
li r.0, 1 // set bit for mask generation
slw r.0, r.0, r.6 // compute ready summary set member
cmpw r.11, r.12 // check for list empty
stw r.11, LsFlink(r.12) // set forward link in previous entry
stw r.12, LsBlink(r.11) // set backward link in next entry
bne ksc120 // if ne, list is not empty
xor r.26, r.26, r.0 // clear ready summary bit
stw r.26, 0(r.27) // update ready summary
ksc120:
// Swap context to the next thread.
#if !defined(NT_UP)
lwz r.27,[toc]KiContextSwapLock(r.2)// get &KiContextSwapLock
lwz r.28,[toc]KiDispatcherLock(r.2) // get &KiDispatcherLock
#endif
stw NTH, PbCurrentThread(rPrcb) // set new thread current
ksc130:
bl ..SwapContext // swap context
// Lower IRQL and return wait completion status.
// N.B. SwapContext releases the dispatcher database lock.
lbz r.4, ThWaitIrql(NTH) // get original wait IRQL
DISABLE_INTERRUPTS(r.6, r.7)
lhz r.5, KiPcr+PcSoftwareInterrupt(r.0) // check for pending s/w ints
lwz r.14, kscLR(r.sp) // get return address
lwz r.31, ThWaitStatus(NTH) // get wait completion status
lwz r.26, swFrame + ExGpr26(r.sp) // restore gprs 26 thru 30
lwz r.27, swFrame + ExGpr27(r.sp) //
lwz r.28, swFrame + ExGpr28(r.sp) //
lwz r.29, swFrame + ExGpr29(r.sp) //
lwz r.30, swFrame + ExGpr30(r.sp) //
stb r.4, KiPcr+PcCurrentIrql(r.0) // set new IRQL
ENABLE_INTERRUPTS(r.6)
cmpw r.5, r.4 // see if s/w int could now run
bgtl- ..KiDispatchSoftwareInterrupt // jif pending int can run
mtlr r.14 // set return address
ori r.3, r.31, 0 // set return status
lwz r.31, swFrame + ExGpr31(r.sp) // restore r.31
lwz r.14, swFrame + ExGpr14(r.sp) // restore r.14
addi r.sp, r.sp, kscFrameLength // return stack frame
SPECIAL_EXIT(KiSwapThread)
kscIdle:
// All ready queues were scanned without finding a runnable thread so
// default to the idle thread and set the appropriate bit in idle summary.
lwz r.5, [toc]KiIdleSummary(r.toc) // get &KiIdleSummary
#if defined(NT_UP)
li r.4, 1 // set current idle summary
#else
lwz r.4, 0(r.5) // get current idle summary
or r.4, r.4, r.3 // set member bit in idle summary
#endif
stw r.4, 0(r.5) // set new idle summary
lwz NTH,PbIdleThread(rPrcb) // set address of idle thread
b ksc120 // switch to idle thread
SBTTL("Swap Context to Next Thread")
// Routine Description:
// This routine is called to swap context from one thread to the next.
// Arguments:
// r.sp Pointer to { StackFrameHeader, ExceptionFrame }
// r.31 NTH Address of next thread object
// r.30 OTH Address of previous thread object
// r.29 rPrcb Address of processor control block
// r.28 Address of KiDispatcherLock
// r.27 Address of KiContextSwapLock
// Return value:
// r.31 NTH Address of current thread object.
// r.29 rPrcb Address of processor control block
// Note that the TOC register is neither saved nor restored across a
// thread switch. This is because we are in NTOSKRNL (actually in the
// routine SwapContext) in both threads (ie the TOC does not change).
// GPR 13 is set to the address of the TEB where it will remain.
SPECIAL_ENTRY_S(SwapContext,_TEXT$00)
mfcr r.3 // get condition register
// Acquire the context swap lock so the address space of the old process
// cannot be deleted and then release the dispatcher database lock.
// N.B. This lock is used to protect the address space until the context
// switch has sufficiently progressed to the point where the address
// space is no longer needed. This lock is also acquired by the reaper
// thread before it finishes thread termination.
#if !defined(NT_UP)
b LkCtxSw // skip spin code
SPIN_ON_SPIN_LOCK(r.27,r.4,LkCtxSw,LkCtxSwSpin) // spin on context swap lock
ACQUIRE_SPIN_LOCK(r.27,r.31,r.4,LkCtxSw,LkCtxSwSpin) // acquire context swap lock
#endif
// Set the new thread's state to Running before releasing the dispatcher lock.
li r.8, Running // set state of new thread
stb r.8, ThState(NTH) // to running.
#if !defined(NT_UP)
RELEASE_SPIN_LOCK(r.28,r.4) // release dispatcher lock
#endif
// Save old thread non-volatile context.
mflr r.0 // get return address
stw r.15, swFrame + ExGpr15(r.sp) // save gprs 15 thru 25
stw r.16, swFrame + ExGpr16(r.sp) //
stw r.17, swFrame + ExGpr17(r.sp) //
stw r.18, swFrame + ExGpr18(r.sp) //
stw r.19, swFrame + ExGpr19(r.sp) //
stw r.20, swFrame + ExGpr20(r.sp) //
stw r.21, swFrame + ExGpr21(r.sp) //
stw r.22, swFrame + ExGpr22(r.sp) //
stw r.23, swFrame + ExGpr23(r.sp) //
stw r.24, swFrame + ExGpr24(r.sp) //
stw r.25, swFrame + ExGpr25(r.sp) //
stfd f.14, swFrame + ExFpr14(r.sp) // save non-volatile
stfd f.15, swFrame + ExFpr15(r.sp) // floating point regs
stfd f.16, swFrame + ExFpr16(r.sp) //
stfd f.17, swFrame + ExFpr17(r.sp) //
stfd f.18, swFrame + ExFpr18(r.sp) //
stfd f.19, swFrame + ExFpr19(r.sp) //
stfd f.20, swFrame + ExFpr20(r.sp) //
stfd f.21, swFrame + ExFpr21(r.sp) //
stfd f.22, swFrame + ExFpr22(r.sp) //
stfd f.23, swFrame + ExFpr23(r.sp) //
stfd f.24, swFrame + ExFpr24(r.sp) //
stfd f.25, swFrame + ExFpr25(r.sp) //
stfd f.26, swFrame + ExFpr26(r.sp) //
stfd f.27, swFrame + ExFpr27(r.sp) //
stfd f.28, swFrame + ExFpr28(r.sp) //
stfd f.29, swFrame + ExFpr29(r.sp) //
stfd f.30, swFrame + ExFpr30(r.sp) //
stfd f.31, swFrame + ExFpr31(r.sp) //
stw r.3, swFrame + SwConditionRegister(r.sp)// save CR
stw r.0, swFrame + SwSwapReturn(r.sp) // save return address
PROLOGUE_END(SwapContext)
// The following entry point is used to switch from the idle thread to
// another thread.
..SwapFromIdle:
#if DBG
cmpw NTH, OTH
bne th_ok
twi 31, 0, 0x16
th_ok:
#endif
stw NTH, KiPcr+PcCurrentThread(r.0) // set new thread current
// Get the old and new process object addresses.
#define NPROC r.3
#define OPROC r.4
lwz NPROC, ThApcState + AsProcess(NTH) // get new process object
lwz OPROC, ThApcState + AsProcess(OTH) // get old process object
DISABLE_INTERRUPTS(r.15, r.6)
lwz r.13, ThTeb(NTH) // get addr of user TEB
lwz r.24, ThStackLimit(NTH) // get stack limit
lwz r.23, ThInitialStack(NTH) // get initial kernel stk ptr
lwz r.22, ThKernelStack(NTH) // get new thread stk ptr
cmpw cr.0, NPROC, OPROC // same process ?
stw r.sp, ThKernelStack(OTH) // save current kernel stk ptr
stw r.13, KiPcr+PcTeb(r.0) // set addr of user TEB
// Although interrupts are disabled, someone may be attempting to
// single step thru the following. I can't see anyway to perform
// the two operations atomically so I am inserting a label that is
// known externally and can be checked against the exception address
// if we fail stack validation in common_exception_entry (in real0.s)
// in which case it's really ok. This has no performance impact.
// *** WARNING ****** WARNING ****** WARNING ****** WARNING ***
// (1) these two instructions MUST stay together,
// (2) the stack validation code in common_exception_entry
// KNOWS that the second instruction is a 'ori r.sp, r.22, 0'
// and will perform such an instruction in line to correct
// the problem. If you change this sequence you will need
// to make an equivalent change in real0.s and the correct-
// ability is dependent on the second instruction destroying
// the stack pointer.
// (plj).
stw r.24, KiPcr+PcStackLimit(r.0) // set stack limit
stw r.23, KiPcr+PcInitialStack(r.0) // set initial kernel stack ptr
.globl KepSwappingContext
KepSwappingContext:
ori r.sp, r.22, 0 // switch stacks
#if !defined(NT_UP)
// Old process address space is no longer required. Ensure all
// stores are done prior to releasing the ContextSwap lock.
// N.B. SwapContextLock is still needed to ensure KiMasterPid
// integrity.
li r.16, 0
eieio
bne cr.0, ksc10
stw r.16, 0(r.27) // release Context Swap lock
b ksc20
ksc10:
#else
beq cr.0, ksc20
#endif
// If the process sequence number matches the system sequence number, then
// use the process PID. Otherwise, allocate a new process PID.
// N.B. The following code is duplicated from KiSwapProcess and will
// join KiSwapProcess at SwapProcessSlow if sequence numbers
// don't match. Register usage from here to the branch should
// match KiSwapProcess.
lwz r.10,[toc]KiMasterSequence(r.toc) // get &KiMasterSequence
lwz r.9,PrProcessSequence(NPROC) // get process sequence number
lwz r.11,0(r.10) // get master sequence number
lwz r.7,PrProcessPid(NPROC) // get process PID
cmpw r.11,r.9 // master sequence == process sequence?
bne ksc15 // jif not equal, go the slow path
#if !defined(NT_UP)
stw r.16, 0(r.27) // release Context Swap lock
#endif
// Swap address space to the specified process.
lwz r.5,PrDirectoryTableBase(r.3) // get page dir page real addr
mtsr 0,r.7 // set sreg 0
addi r.7,r.7,1 // add 1 to VSID
mtsr 1,r.7 // set sreg 1
addi r.7,r.7,1 // add 1 to VSID
mtsr 2,r.7 // set sreg 2
addi r.7,r.7,1 // add 1 to VSID
mtsr 3,r.7 // set sreg 3
addi r.7,r.7,1 // add 1 to VSID
mtsr 4,r.7 // set sreg 4
addi r.7,r.7,1 // add 1 to VSID
mtsr 5,r.7 // set sreg 5
addi r.7,r.7,1 // add 1 to VSID
mtsr 6,r.7 // set sreg 6
addi r.7,r.7,1 // add 1 to VSID
mtsr 7,r.7 // set sreg 7
addi r.7,r.7,12-7 // add 5 to VSID
mtsr 12,r.7 // set sreg 12
isync // context synchronize
stw r.5,KiPcr+PcPgDirRa(r.0) // store page dir page ra in PCR
#if COLLECT_PAGING_DATA
lwz r.10,[toc]KiFlushOnProcessSwap(r.toc)
lwz r.10,0(r.10)
cmpwi r.10,0
bnel ..KeFlushCurrentTb
#endif
b ksc20
ksc15:
bl SwapProcessSlow
ksc20:
lbz r.5, ThApcState + AsKernelApcPending(NTH)
lbz r.16, ThDebugActive(NTH) // get the active debug register
// mask
stb r.5, KiPcr+PcApcInterrupt(r.0) // set APC pending appropriately
stb r.16, KiPcr+PcDebugActive(r.0) // set the active debug register
// mask for the new thread
lwz r.5, PbContextSwitches(rPrcb) // get context switch count
lwz r.7, ThContextSwitches(NTH)
addi r.5, r.5, 1 // bump context switch count
stw r.5, PbContextSwitches(rPrcb) // for processor.
addi r.7, r.7, 1 // bump context switch count
stw r.7, ThContextSwitches(NTH) // for this thread
ENABLE_INTERRUPTS(r.15)
lwz r.0, swFrame + SwSwapReturn(r.sp) // get return address
lwz r.5, swFrame + SwConditionRegister(r.sp)// get CR
lwz r.15, swFrame + ExGpr15(r.sp) // restore gprs 15 thru 25
lwz r.16, swFrame + ExGpr16(r.sp) //
lwz r.17, swFrame + ExGpr17(r.sp) //
lwz r.18, swFrame + ExGpr18(r.sp) //
lwz r.19, swFrame + ExGpr19(r.sp) //
lwz r.20, swFrame + ExGpr20(r.sp) //
lwz r.21, swFrame + ExGpr21(r.sp) //
lwz r.22, swFrame + ExGpr22(r.sp) //
lwz r.23, swFrame + ExGpr23(r.sp) //
lwz r.24, swFrame + ExGpr24(r.sp) //
lwz r.25, swFrame + ExGpr25(r.sp) //
lfd f.14, swFrame + ExFpr14(r.sp) // restore non-volatile
lfd f.15, swFrame + ExFpr15(r.sp) // floating point regs
lfd f.16, swFrame + ExFpr16(r.sp) //
lfd f.17, swFrame + ExFpr17(r.sp) //
lfd f.18, swFrame + ExFpr18(r.sp) //
lfd f.19, swFrame + ExFpr19(r.sp) //
lfd f.20, swFrame + ExFpr20(r.sp) //
lfd f.21, swFrame + ExFpr21(r.sp) //
lfd f.22, swFrame + ExFpr22(r.sp) //
lfd f.23, swFrame + ExFpr23(r.sp) //
lfd f.24, swFrame + ExFpr24(r.sp) //
lfd f.25, swFrame + ExFpr25(r.sp) //
lfd f.26, swFrame + ExFpr26(r.sp) //
lfd f.27, swFrame + ExFpr27(r.sp) //
mtlr r.0 // set return address
mtcrf 0xff, r.5 // set condition register
lfd f.28, swFrame + ExFpr28(r.sp) //
lfd f.29, swFrame + ExFpr29(r.sp) //
lfd f.30, swFrame + ExFpr30(r.sp) //
lfd f.31, swFrame + ExFpr31(r.sp) //
SPECIAL_EXIT(SwapContext)
#undef NTH
#undef OTH
// VOID
// KiSwapProcess (
// IN PKPROCESS NewProcess,
// IN PKPROCESS OldProcess
// )
// Routine Description:
// This function swaps the address space from one process to another by
// moving to the PCR the real address of the process page directory page
// and loading segment registers 0-7 and 12 with VSIDs derived therefrom.
// The fast path below is duplicated inline in SwapContext for speed.
// SwapContext joins this code at SwapProcessSlow if sequence numbers
// differ.
// Arguments:
// NewProcess (r3) - Supplies a pointer to a control object of type process
// which represents the new process that is switched to.
// OldProcess (r4) - Supplies a pointer to a control object of type process
// which represents the old process that is switched from.
// Return Value:
// None.
LEAF_ENTRY_S(KiSwapProcess,_TEXT$00)
// Get the Context Swap lock. This lock is used to protect a
// processes memory space, it serves double duty to protect access
// to KiMasterSequence.
// N.B. It is already held if entry is via SwapProcessSlow, the
// lock is ALWAYS released by this routine.
#if !defined(NT_UP)
lwz r.6, [toc]KiContextSwapLock(r.2)
ACQUIRE_SPIN_LOCK(r.6,r.3,r.5,LkCtxSw2,LkCtxSw2Spin)
#endif
// If the process sequence number matches the system sequence number, then
// use the process PID. Otherwise, allocate a new process PID.
// WARNING: if you change register usage in the following be sure to make
// the same changes in SwapContext.
lwz r.10,[toc]KiMasterSequence(r.toc) // get &KiMasterSequence
lwz r.9,PrProcessSequence(r.3) // get process sequence number
lwz r.11,0(r.10) // get master sequence number
lwz r.7,PrProcessPid(r.3) // get process PID
cmpw r.11,r.9 // master sequence == process sequence?
bne SwapProcessSlow // jif not equal, out of line
// Swap address space to the specified process.
spup: lwz r.5,PrDirectoryTableBase(r.3) // get page dir page real addr
DISABLE_INTERRUPTS(r.8,r.0) // disable interrupts
#if !defined(NT_UP)
sync
li r.10, 0
stw r.10, 0(r.6) // release KiContextSwapLock
#endif
stw r.5,KiPcr+PcPgDirRa(r.0) // store page dir page ra in PCR
mtsr 0,r.7 // set sreg 0
addi r.7,r.7,1 // add 1 to VSID
mtsr 1,r.7 // set sreg 1
addi r.7,r.7,1 // add 1 to VSID
mtsr 2,r.7 // set sreg 2
addi r.7,r.7,1 // add 1 to VSID
mtsr 3,r.7 // set sreg 3
addi r.7,r.7,1 // add 1 to VSID
mtsr 4,r.7 // set sreg 4
addi r.7,r.7,1 // add 1 to VSID
mtsr 5,r.7 // set sreg 5
addi r.7,r.7,1 // add 1 to VSID
mtsr 6,r.7 // set sreg 6
addi r.7,r.7,1 // add 1 to VSID
mtsr 7,r.7 // set sreg 7
addi r.7,r.7,12-7 // add 5 to VSID
mtsr 12,r.7 // set sreg 12
isync // context synchronize
ENABLE_INTERRUPTS(r.8) // enable interrupts
#if COLLECT_PAGING_DATA
lwz r.10,[toc]KiFlushOnProcessSwap(r.toc)
lwz r.10,0(r.10)
cmpwi r.10,0
bne ..KeFlushCurrentTb
#endif
ALTERNATE_EXIT(KiSwapProcess) // return
// We need a new PID, the dispatcher database lock is still held so
// we can update KiMasterPid without further protection.
SwapProcessSlow:
lwz r.8,[toc]KiMasterPid(r.toc) // get &KiMasterPid
lwz r.7,0(r.8) // get KiMasterPid
addi r.7,r.7,16 // bump master pid
rlwinm. r.7,r.7,0,0x007ffff0 // detect PID wrap
beq ..KxSwapProcess // jif PID wrap
stw r.11,PrProcessSequence(r.3) // save new process sequence
// control returns here from KxSwapProcess
spnp:
#if !defined(NT_UP)
lwz r.6, [toc]KiContextSwapLock(r.2)
#endif
stw r.7,0(r.8) // save new master PID
stw r.7,PrProcessPid(r.3) // save new process PID
b spup // continue with main line code
#if !defined(NT_UP)
SPIN_ON_SPIN_LOCK(r.6,r.5,LkCtxSw2,LkCtxSw2Spin)
#endif
DUMMY_EXIT(KiSwapProcess)
// VOID
// KxSwapProcess (
// IN PKPROCESS NewProcess,
// )
// Routine Description:
// This function is called (only) from KiSwapProcess when PID wrap has
// occured. KiSwapProcess is a LEAF function. The purpose of this
// function is to alloacte a stack frame and save data that needs to
// be restored for KiSwapProcess. This routine is called aproximately
// once every 16 million new processes. The emphasis in KiSwapProcess
// is to handle the other 16 million - 1 cases as fast as possible.
// Arguments:
// NewProcess (r3) - Supplies a pointer to a control object of
// type process which represents the new process being switched to.
// This must be saved and restored for KiSwapProcess.
// &KiMasterPid (r8) - Address of system global KiMasterPid
// This must be restored for KiSwapProcess.
// &KiMasterSequence (r10) - Address of system global KiMasterSequence.
// KiMasterSequence (r11) - Current Value of the above variable.
// Return Value:
// None.
// Registers r3, r8 and the Link Register are restored. r7 contains
// the new PID which will be 16.
.struct 0
.space StackFrameHeaderLength
spLR: .space 4 // link register save
spR3: .space 4 // new process address save
.align 3 // ensure correct alignment
spFrameLength:
SPECIAL_ENTRY_S(KxSwapProcess,_TEXT$00)
mflr r.0 // get link register
stwu r.sp,-spFrameLength(r.sp) // buy stack frame
stw r.3,spR3(r.sp) // save new process address
stw r.0,spLR(r.sp) // save swap process' return address
PROLOGUE_END(KxSwapProcess)
// PID wrap has occured. On PowerPC we do not need to lock the process
// id wrap lock because tlb synchronization is handled by hardware.
addic. r.11,r.11,1 // bump master sequence number
bne+ spnsw // jif sequence number did not wrap
// The master sequence number has wrapped, this is 4 billion * 16 million
// processes,... not too shabby. We start the sequence again at 2 in case
// there are system processes that have been running since the system first
// started.
li r.11,2 // start again at 2
spnsw: stw r.11,0(r.10) // save new master sequence number
stw r.11,PrProcessSequence(r.3) // save new process sequence num
bl ..KeFlushCurrentTb // flush entire HPT (and all processor
// TLBs)
lwz r.0,spLR(r.sp) // get swap process' return address
lwz r.3,spR3(r.sp) // get new process address
lwz r.8,[toc]KiMasterPid(r.toc) // get &KiMasterPid
addi r.sp,r.sp,spFrameLength // return stack frame
li r.7,16 // set new PID
mtlr r.0
b spnp // continue in KiSwapProcess
DUMMY_EXIT(KxSwapProcess)
// VOID
// KiIdleLoop (
// VOID
// )
// Routine Description:
// This is the idle loop for NT. This code runs in a thread for
// each processor in the system. The idle thread runs at IRQL
// DISPATCH_LEVEL and polls for work.
// Arguments:
// None.
// Return Value:
// None. (This routine never returns).
// Non-volatile register usage is as follows.
// r.14 --unused - available --
// r.15 Address of KdDebuggerEnabled
// r.16 Kernel TOC (backup)
// r.17 Idle loop MSR with Interrupts DISABLED
// r.18 Idle loop MSR with Interrupts ENABLED
// r.19 HalProcessorIdle entry point
// r.20 HAL's TOC
// r.21 Debugger poll count
// r.22 Address of KeTickCount
// r.23 Zero
// r.24 Address of dispatcher database lock (MP) (backup for r28)
// r.25 DpcListHead
// r.26 --unused - available --
// r.27 Address of Context Swap lock
// r.28 Address of dispatcher database lock (MP)
// r.29 Address of Processor Control Block
// When another thread is selected to run, SwapContext is called.
// Normally, callers of SwapContext are responsible for saving and
// restoring non-volatile regs r.14 and r.26 thru r.31. SwapContext
// saves/restores gprs r.15 thru r.25. The idle loop never returns so
// previous contents of r.14 and r.26 thru r.31 are not saved. The
// idle loop pre-initializes the storage area where SwapContext would
// normally save r.15 thru r.25 with values that the idle loop needs
// in those registers upon return from SwapContext and skips over the
// register save on the way into SwapContext (alternate entry point
// SwapFromIdle). All callers to SwapContext pass the following
// arguments-
// r.27 Address of Context Swap lock (&KiContextSwapLock)
// r.28 Address of dispatcher database lock (&KiDispatcherLock)
// r.29 Address of PRCB
// r.30 Address of OLD thread object
// r.31 Address of NEW thread object
// The idle loop does not have a fixed use for regs r.30 and r.31.
// r.29 contains the correct value for this processor. r.14 and
// r.26 contents are unknown and must be regenerated upon return
// from SwapContext. The assignment of function to these registers
// was chosen for easy regeneration of content.
// Note also that r.21 was assigned in the range of registers
// restored by SwapContext so that it is reset to its initial
// values whenever SwapContext is called.
#define rDbg r.15
#define rKTOC r.16
#define rIntOff r.17
#define rIntOn r.18
#define rHalIdle r.19
#define rHalToc r.20
#define rDbgCount r.21
#define rTickP r.22
#define rZero r.23
#define rDispLkSave r.24
#define rDPCHEAD r.25
#define rDispLk r.28
#define rPrcb r.29
SPECIAL_ENTRY_S(KiIdleLoop,_TEXT$00)
mflr r.0 // get return address
stwu r.sp, -kscFrameLength(r.sp) // buy stack frame
stw r.0, kscLR(r.sp) // save return address
PROLOGUE_END(KiIdleLoop)
// Setup initial global register values
ori rKTOC, r.toc, 0 // backup kernel's TOC
lwz rPrcb, KiPcr+PcPrcb(r.0) // Address of PCB to rPrcb
lwz rTickP, [toc]KeTickCount(r.toc) // Address of KeTickCount
lwz rDbg, [toc]KdDebuggerEnabled(r.toc)// Addr KdDebuggerEnabled
lwz rHalToc,[toc]__imp_HalProcessorIdle(r.toc)
lwz rHalToc,0(rHalToc)
lwz rHalIdle,0(rHalToc) // HalProcessorIdle entry
lwz rHalToc,4(rHalToc) // HAL's TOC
li rZero, 0 // Keep zero around, we use it
mfmsr rIntOff // get current machine state
rlwinm rIntOff, rIntOff, 0, 0xffff7fff // clear interrupt enable
ori rIntOn, rIntOff, 0x00008000 // set interrupt enable
#if !defined(NT_UP)
lwz rDispLk, [toc]KiDispatcherLock(r.toc)// get &KiDispatcherLock
lwz r.27, [toc]KiContextSwapLock(r.toc) // get &KiContextSwapLock
#endif
addi rDPCHEAD, rPrcb, PbDpcListHead // compute DPC listhead address
li rDbgCount, 0 // Clear breakin loop counter
#if !defined(NT_UP)
ori rDispLkSave, rDispLk, 0 // copy &KiDispatcherLock
#endif
// Registers 15 thru 25 are normally saved by SwapContext but the idle
// loop uses an alternate entry that skips the save by SwapContext.
// SwapContext will still restore them so we set up the stack so what
// we want is what gets restored. This is especially useful for things
// whose values need to be reset after SwapContext is called, eg rDbgCount.
lwz r.4, [toc]__imp_KeLowerIrql(r.toc) // &&fd(KeLowerIrql)
stw r.15, swFrame + ExGpr15(r.sp)
stw r.16, swFrame + ExGpr16(r.sp)
stw r.17, swFrame + ExGpr17(r.sp)
lwz r.4, 0(r.4) // &fd(KeLowerIrql)
stw r.18, swFrame + ExGpr18(r.sp)
stw r.19, swFrame + ExGpr19(r.sp)
stw r.20, swFrame + ExGpr20(r.sp)
lwz r.5, 0(r.4) // &KeLowerIrql
stw r.21, swFrame + ExGpr21(r.sp)
stw r.22, swFrame + ExGpr22(r.sp)
stw r.23, swFrame + ExGpr23(r.sp)
stw r.24, swFrame + ExGpr24(r.sp)
stw r.25, swFrame + ExGpr25(r.sp)
// Control reaches here with IRQL at HIGH_LEVEL. Lower IRQL to
// DISPATCH_LEVEL and set wait IRQL of idle thread.
mtctr r.5
lwz r.toc, 4(r.4) // HAL's TOC
lwz r.11, KiPcr+PcCurrentThread(r.0) // Lower thread and processor
li r.3, DISPATCH_LEVEL // IRQL to DISPATCH_LEVEL.
stb r.3, ThWaitIrql(r.11)
bctrl
ori r.toc, rKTOC, 0 // restore our TOC
// In a multiprocessor system the boot processor proceeds directly into
// the idle loop. As other processors start executing, however, they do
// not directly enter the idle loop and spin until all processors have
// been started and the boot master allows them to proceed.
#if !defined(NT_UP)
lwz r.4, [toc]KiBarrierWait(r.toc)
BarrierWait:
lwz r.3, 0(r.4) // get barrier wait value
cmpwi r.3, 0 // if ne spin until allowed
bne BarrierWait // to proceed.
lbz r.3, PbNumber(rPrcb) // get processor number
cmpwi cr.4, r.3, 0 // save "processor == 0 ?"
#endif
// Set condition register and swap return values in the swap frame.
mfcr r.3 // save condition register
stw r.3, swFrame + SwConditionRegister(r.sp)
bl FindIdleReturn
FindIdleReturn:
mflr r.3
addi r.3, r.3, KiIdleReturn - FindIdleReturn
stw r.3, swFrame + SwSwapReturn(r.sp)// save return address
// The following loop is executed for the life of the system.
IdleLoop:
#if DBG
#if !defined(NT_UP)
bne cr.4, CheckDpcList // if ne, not processor zero
#endif
// Check if the debugger is enabled, and whether it is time to poll
// for a debugger breakin. (This check is only performed on cpu 0).
subic. rDbgCount, rDbgCount, 1 // decrement poll counter
bge+ CheckDpcList // jif not time yet.
lbz r.3, 0(rDbg) // check if debugger enabled
li rDbgCount, 20 * 1000 // set breakin loop counter
cmpwi r.3, 0
beq+ CheckDpcList // jif debugger not enabled
bl ..KdPollBreakIn // check if breakin requested
cmpwi r.3, 0
beq+ CheckDpcList // jif no breakin request
li r.3, DBG_STATUS_CONTROL_C // send status to debugger
bl ..DbgBreakPointWithStatus
#endif
// Disable interrupts and check if there is any work in the DPC list.
CheckDpcList:
mtmsr rIntOn // give interrupts a chance
isync // to interrupt spinning
mtmsr rIntOff // disable interrupts
cror 0,0,0 // N.B. 603e/ev Errata 15
// Process the deferred procedure call list for the current processor.
lwz r.3, LsFlink(rDPCHEAD) // get address of first entry
cmpw r.3, rDPCHEAD // is list empty?
beq CheckNextThread // if eq, DPC list is empty
ori r.31, rDPCHEAD, 0
bl ..KiProcessDpcList // process the DPC list
// Clear dispatch interrupt pending.
stb rZero, KiPcr+PcDispatchInterrupt(r.0) // clear pending DPC interrupt
#if DBG
li rDbgCount, 0 // clear breakin loop counter
#endif
// Check if a thread has been selected to run on this processor
CheckNextThread:
lwz r.31, PbNextThread(rPrcb) // get address of next thread
cmpwi r.31, 0
beq Idle // jif no thread to execute
// A thread has been selected for execution on this processor. Acquire
// dispatcher database lock, get the thread address again (it may have
// changed), clear the address of the next thread in the processor block,
// and call swap context to start execution of the selected thread.
// N.B. If the dispatcher database lock cannot be obtained immediately,
// then attempt to process another DPC rather than spinning on the
// dispatcher database lock.
#if !defined(NT_UP)
TRY_TO_ACQUIRE_SPIN_LOCK(rDispLk, rDispLk, r.11, LkDisp, CheckDpcList)
#endif
mtmsr rIntOn // enable interrupts
cror 0,0,0 // N.B. 603e/ev Errata 15
#if !defined(NT_UP)
lwz r.31, PbNextThread(rPrcb) // get next thread address
#endif
lwz r.30, PbCurrentThread(rPrcb) // get current thread address
stw rZero, PbNextThread(rPrcb) // clear address of next thread
stw r.31, PbCurrentThread(rPrcb) // set new thread current
// Acquire the context swap lock so the address space of the old process
// cannot be deleted and then release the dispatcher database lock. In
// this case the old process is the system process, but the context swap
// code releases the context swap lock so it must be acquired.
// N.B. This lock is used to protect the address space until the context
// switch has sufficiently progressed to the point where the address
// space is no longer needed. This lock is also acquired by the reaper
// thread before it finishes thread termination.
#if !defined(NT_UP)
ACQUIRE_SPIN_LOCK(r.27,r.31,r.3,LkCtxSw1,LkCtxSw1Spin)
#endif
// Set the new thread's state to Running before releasing the dispatcher lock.
li r.3, Running // set state of new thread
stb r.3, ThState(r.31) // to running.
#if !defined(NT_UP)
RELEASE_SPIN_LOCK(r.28,rZero)
#endif
bl ..SwapFromIdle // swap context to new thread
KiIdleReturn:
#if !defined(NT_UP)
ori rDispLk, rDispLkSave, 0 // restore &KiDispatcherLock
// rDbgCount (r.21) will have been reset to 0 by the register restore
// at the end of SwapContext.
// If processor 0, check for debugger breakin, otherwise just check for
// DPCs again.
#endif
b IdleLoop
// There are no entries in the DPC list and a thread has not been selected
// for execution on this processor. Call the HAL so power managment can be
// performed.
// N.B. The HAL is called with interrupts disabled. The HAL will return
// with interrupts enabled.
Idle:
mtctr rHalIdle // set entry point
ori r.toc, rHalToc, 0 // set HAL's TOC
bctrl // call HalProcessorIdle
isync // give HAL's interrupt enable
// a chance to take effect
ori r.toc, rKTOC, 0 // restore ntoskrnl's TOC
b IdleLoop
#if !defined(NT_UP)
SPIN_ON_SPIN_LOCK(r.27,r.3,LkCtxSw1,LkCtxSw1Spin)
#endif
DUMMY_EXIT(KiIdleLoop)
#undef rDPCHEAD
SBTTL("Process Deferred Procedure Call List")
// Routine Description:
// This routine is called to process the given deferred procedure call
// list.
// If called from KiDispatchInterrupt, we will have been switched to
// the interrupt stack, the new stack pointer is in r.sp and entry is
// at ..KiProcessDpcList.alt.
// If called from the idle loop, we run on the idle loop thread's
// stack and no special action is needed. However, the idle loop
// does not pass r.9 as we expect it and only passes r.0 = 0 on MP
// systems. We take advantage of the separate entry points to set
// these registers appropriately.
// N.B. Interrupts must be disabled on entry to this routine. Control
// is returned to the caller with the same conditions true.
// Arguments:
// None.
// On entry:
// r.0 - Zero
// r.9 - Machine State Register prior to disabling interrupts.
// rPrcb r.29 - address of processor control block.
// r.31 - address of DPC list head.
// On exit:
// r.0 - Zero
// r.9 - Machine State Register prior to disabling interrupts.
// rPrcb r.29 - address of processor control block.
// r.31 - address of DPC list head.
// Return value:
// None.
.struct 0
.space StackFrameHeaderLength
dp.lr: .space 4
dp.toc: .space 4
#if DBG
dp.func:.space 4
dp.strt:.space 4
dp.cnt: .space 4
dp.time:.space 4
#endif
.align 3 // ensure stack frame length is multile of 8
dp.framelen:
.text
#if DBG
# define rDpStart r.22
# define rDpCount r.23
# define rDpTime r.24
#endif
SPECIAL_ENTRY_S(KiProcessDpcList,_TEXT$00)
stwu r.sp, -dp.framelen(r.sp) // buy stack frame
// see routine description for why we do the following.
ori r.9, rIntOn, 0 // get MSR interrupts enabled
#if defined(NT_UP)
li r.0, 0
#endif
..KiProcessDpcList.alt:
mflr r.7 // save return address
// save regs we will use,... don't need to save 29 and 31 as they
// were saved by our caller and currently contain the values we want.
stw r.toc, dp.toc(r.sp)
#if DBG
stw rDpTime, dp.time(r.sp)
stw rDpCount, dp.cnt(r.sp)
stw rDpStart, dp.strt(r.sp)
#endif
stw r.7, dp.lr(r.sp) // save Link Register
PROLOGUE_END(KiProcessDpcList)
DpcCallRestart:
stw r.sp, PbDpcRoutineActive(rPrcb) // set DPC routine active
// Process the DPC list.
DpcCall:
#if !defined(NT_UP)
addi r.7, rPrcb, PbDpcLock // compute DPC lock address
ACQUIRE_SPIN_LOCK(r.7, r.7, r.0, spinlk2, spinlk2spin)
#endif
lwz r.3, LsFlink(r.31) // get address of first entry
lwz r.12, LsFlink(r.3) // get addr of next entry
cmpw r.3, r.31 // is list empty?
subi r.3, r.3, DpDpcListEntry // subtract DpcListEntry offset
beq- UnlkDpc0 // if yes, release the lock.
// Get deferred routine address, this is done early as what
// we actually have is a function descriptor's address and we
// need to get the entry point address.
lwz r.11, DpDeferredRoutine(r.3)
lwz r.8, PbDpcQueueDepth(rPrcb) // get DPC queue depth
// remove entry from list
stw r.12, LsFlink(r.31) // set addr of next in header
stw r.31, LsBlink(r.12) // set addr of previous in next
lwz r.10, 0(r.11) // get DPC code address
// entry removed, set up arguments for DPC proc
// args are-
// dpc object address (r.3)
// deferred context (r.4)
// system argument 1 (r.5)
// system argument 2 (r.6)
// note, the arguments must be loaded from the DPC object BEFORE
// the inserted flag is cleared to prevent the object being
// overwritten before its time.
lwz r.4, DpDeferredContext(r.3)
lwz r.5, DpSystemArgument1(r.3)
lwz r.6, DpSystemArgument2(r.3)
subi r.8, r.8, 1 // decrement DPC queue depth
stw r.8, PbDpcQueueDepth(rPrcb) //
stw r.0, DpLock(r.3) // clear DPC inserted state
#if !defined(NT_UP)
RELEASE_SPIN_LOCK(r.7, r.0)
#endif
mtctr r.10 // ready address for branch
ENABLE_INTERRUPTS(r.9)
lwz r.toc, 4(r.11) // get DPC toc pointer
#if DBG
lwz rDpStart, KiPcr2 + Pc2TickCountLow(r.0) // get current time
lwz rDpCount, PbInterruptCount(rPrcb)// get current interrupt count
lwz rDpTime, PbInterruptTime(rPrcb) // get current interrupt time
stw r.10, dp.func(r.sp)
#endif
bctrl // call DPC routine
li r.0, 0 // reset zero constant
#if DBG
lbz r.10, KiPcr+PcCurrentIrql(r.0) // get current IRQL
cmplwi r.10, DISPATCH_LEVEL // check if < DISPATCH_LEVEL
blt DpcBadIrql // jif IRQL < DISPATCH_LEVEL
DpcIrqlOk:
lwz r.12, KiPcr2 + Pc2TickCountLow(r.0) // calculate time spent in
sub r.12, r.12, rDpStart // r.12 = time
cmpwi r.12, 100
bge DpcTookTooLong // jif >= 1 second
DpcTimeOk:
#endif
// Check to determine if any more DPCs are available to process.
DISABLE_INTERRUPTS(r.9, r.10)
lwz r.3, LsFlink(r.31) // get address of first entry
cmpw r.3, r.31 // is list empty?
bne- DpcCall // if no, process it
// Clear DpcRoutineActive, then check one last time that the DPC queue is
// empty. This is required to close a race condition with the DPC queueing
// code where it appears that a DPC routine is active (and thus an
// interrupt is not requested), but this code has decided that the queue
// is empty and is clearing DpcRoutineActive.
stw r.0, PbDpcRoutineActive(rPrcb)
stw r.0, PbDpcInterruptRequested(rPrcb) // clear DPC interrupt requested
eieio // force writes out
lwz r.3, LsFlink(r.31)
cmpw r.3, r.31
bne- DpcCallRestart
DpcDone:
// List is empty, restore non-volatile registers we have used.
lwz r.10, dp.lr(r.sp) // get link register
#if DBG
lwz rDpTime, dp.time(r.sp)
lwz rDpCount, dp.cnt(r.sp)
lwz rDpStart, dp.strt(r.sp)
#endif
// Return to caller.
lwz r.toc, dp.toc(r.sp) // restore kernel toc
mtlr r.10 // set return address
lwz r.sp, 0(r.sp) // release stack frame
blr // return
UnlkDpc0:
// The DPC list became empty while we were acquiring the DPC queue lock.
// Clear DPC routine active. The race condition mentioned above doesn't
// exist here because we hold the DPC queue lock.
stw r.0, PbDpcRoutineActive(rPrcb)
#if !defined(NT_UP)
RELEASE_SPIN_LOCK(r.7, r.0)
#endif
b DpcDone
// DpcTookTooLong, DpcBadIrql
// Come here is it took >= 1 second to execute a DPC routine. This is way
// too long, assume something is wrong and breakpoint.
// This code is out of line to avoid wasting cache space for something that
// (hopefully) never happens.
#if DBG
DpcTookTooLong:
lwz r.toc, dp.toc(r.sp) // restore kernel's TOC
lwz r.11, PbInterruptCount(rPrcb) // get current interrupt count
lwz r.10, PbInterruptTime(rPrcb) // get current interrupt time
lwz r.toc, dp.toc(r.sp) // restore our toc
sub r.11, r.11, rDpCount // compute number of interrupts
sub r.10, r.10, rDpTime // compute interrupt time
bl ..DbgBreakPoint // execute debug breakpoint
b DpcTimeOk // continue
DpcBadIrql:
lwz r.toc, dp.toc(r.sp) // restore kernel's TOC
bl ..DbgBreakPoint // breakpoint
b DpcIrqlOk // continue
#endif
#if !defined(NT_UP)
SPIN_ON_SPIN_LOCK(r.7, r.11, spinlk2, spinlk2spin)
#endif
DUMMY_EXIT(KiProcessDpcList)
SBTTL("Dispatch Interrupt")
#define rDPCHEAD r.31
// Routine Description:
// This routine is entered as the result of a software interrupt generated
// at DISPATCH_LEVEL. Its function is to process the Deferred Procedure Call
// (DPC) list, and then perform a context switch if a new thread has been
// selected for execution on the processor.
// This routine is entered at IRQL DISPATCH_LEVEL with the dispatcher
// database unlocked. When a return to the caller finally occurs, the
// IRQL remains at DISPATCH_LEVEL, and the dispatcher database is still
// unlocked.
// Arguments:
// None.
// Outputs:
// ( for call to KiProcessDpcList )
// r.3 - address of first dpc in list
// r.0 - Zero
// r.9 - Machine State Register prior to disabling interrupts.
// rPrcb r.29 - address of processor control block.
// rDPCHEAD r.31 - address of DPC listhead.
// r.9 Machine State Register prior to disabling interrupts.
// ( for call to KiDispIntSwapContext )
// r.28 pointer to Dispatcher Database Lock
// r.29 rPrcb pointer to the processor control block
// r.31 NTH pointer to new thread
// Return Value:
// None.
.struct 0
.space StackFrameHeaderLength
di.lr: .space 4
di.28: .space 4
di.29: .space 4
di.30: .space 4
di.31: .space 4
.align 3 // ensure stack frame length is multile of 8
di.framelen:
SPECIAL_ENTRY_S(KiDispatchInterrupt,_TEXT$00)
mflr r.0
stwu r.sp, -di.framelen(r.sp)
stw r.29, di.29(r.sp)
lwz rPrcb,KiPcr+PcPrcb(r.0)
stw r.30, di.30(r.sp)
stw r.31, di.31(r.sp)
stw r.0, di.lr(r.sp)
stw r.28, di.28(r.sp)
PROLOGUE_END(KiDispatchInterrupt)
// Setup commonly used constants
lwz r.3, PbDpcBypassCount(rPrcb) // get DPC bypass count
li r.0, 0 // zero
addi rDPCHEAD, rPrcb, PbDpcListHead // compute DPC listhead address
addi r.3, r.3, 1 // increment DPC bypass count
stw r.3, PbDpcBypassCount(rPrcb) // store new DPC bypass count
// Process the deferred procedure call list.
PollDpcList:
DISABLE_INTERRUPTS(r.9, r.8)
lwz r.3, LsFlink(rDPCHEAD) // get address of first entry
stb r.0, KiPcr+PcDispatchInterrupt(r.0)
cmpw r.3, rDPCHEAD // list has entries?
beq- di.empty // jif list is empty
// Switch to the interrupt stack
lwz r.6, KiPcr+PcInterruptStack(r.0)// get addr of interrupt stack
lwz r.28, KiPcr+PcInitialStack(r.0) // get current stack base
lwz r.30, KiPcr+PcStackLimit(r.0) // get current stack limit
subi r.4, r.6, KERNEL_STACK_SIZE // compute stack limit
stw r.sp,KiPcr+PcOnInterruptStack(r.0)// flag ON interrupt stack
stw r.sp, -dp.framelen(r.6) // save new back pointer
// N.B. Can't step thru the next two instructions.
stw r.4, KiPcr+PcStackLimit(r.0) // set stack limit
stw r.6, KiPcr+PcInitialStack(r.0) // set current base to int stk
subi r.sp, r.6, dp.framelen // calc new sp
bl ..KiProcessDpcList.alt // process all DPCs for this
// processor.
// N.B. KiProcessDpcList left r.0, r.9 intact.
// Return from KiProcessDpcList switched back to the proper stack,
// update PCR to reflect this.
stw r.30, KiPcr+PcStackLimit(r.0) // restore stack limit
stw r.28, KiPcr+PcInitialStack(r.0) // set old stack current
stw r.0, KiPcr+PcOnInterruptStack(r.0)// clear ON interrupt stack
di.empty:
ENABLE_INTERRUPTS(r.9)
// Check to determine if quantum end has occurred.
// N.B. If a new thread is selected as a result of processing a quantum
// end request, then the new thread is returned with the dispatcher
// database locked. Otherwise, NULL is returned with the dispatcher
// database unlocked.
lwz r.3, KiPcr+PcQuantumEnd(r.0) // get quantum end indicator
cmpwi r.3, 0 // if 0, no quantum end request
beq di.CheckForNewThread
stw r.0, KiPcr+PcQuantumEnd(r.0) // clear quantum end indicator
bl ..KiQuantumEnd // process quantum end
cmpwi r.3, 0 // new thread selected?
li r.0, 0 // reset r.0 to zero
// If KiQuantumEnd returned no new thread to run, the dispatcher
// database is unlocked, get out.
beq+ di.exit
#if !defined(NT_UP)
// Even though the dispatcher database is already locked, we are expected
// to pass the address of the lock in r.28.
lwz r.28, [toc]KiDispatcherLock(r.toc)// get &KiDispatcherLock
#endif
b di.Switch // switch to new thread
// Check to determine if a new thread has been selected for execution on
// this processor.
di.CheckForNewThread:
lwz r.3, PbNextThread(rPrcb) // get address of next thread
cmpwi r.3, 0 // is there a new thread?
beq di.exit // no, branch.
#if !defined(NT_UP)
lwz r.28, [toc]KiDispatcherLock(r.toc)// get &KiDispatcherLock
// Lock dispatcher database and reread address of next thread object since it
// is possible for it to change in a multiprocessor system. (leave address
// of lock in r.28).
TRY_TO_ACQUIRE_SPIN_LOCK(r.28, r.28, r.11, di.spinlk, PollDpcList)
#endif
di.Switch:
lwz r.31, PbNextThread(rPrcb) // get thread address (again)
stw r.0, PbNextThread(rPrcb) // clear addr of next thread obj
// Reready current thread for execution and swap context to the
// selected thread. We do this indirectly thru KiDispIntSwapContext
// to avoid saving and restoring so many registers for the cases
// when KiDispatchInterrupt does not thread switch.
bl ..KiDispIntSwapContext // swap to new thread
di.exit:
lwz r.0, di.lr(r.sp)
lwz r.31, di.31(r.sp)
lwz r.30, di.30(r.sp)
mtlr r.0
lwz r.29, di.29(r.sp)
lwz r.28, di.28(r.sp)
addi r.sp, r.sp, di.framelen
SPECIAL_EXIT(KiDispatchInterrupt)
// VOID
// KiDispIntSwapContext (
// IN PKTHREAD Thread
// )
// Routine Description:
// This routine is called to perform a context switch to the specified
// thread. The current (new previous) thread is re-readied for execution.
// Since this routine is called as subroutine all volatile registers are
// considered free.
// Our caller has saved and will restore gprs 28 thru 31 and does not
// care if we trash them.
// This routine is entered at IRQL DISPATCH_LEVEL with the dispatcher
// database locked. When a return to the caller finally occurs, the
// dispatcher database is unlocked.
// Arguments:
// r.28 pointer to Dispatcher Database Lock
// r.29 rPrcb pointer to the processor control block
// r.31 NTH pointer to new thread
// Outputs: ( for call to SwapContext )
// r.27 pointer to KiContextSwapLock
// r.28 pointer to Dispatcher Database Lock
// r.29 rPrcb pointer to processor control block
// r.30 OTH pointer to old thread
// r.31 NTH pointer to new thread
// Return Value:
// Wait completion status (r.3).
.struct 0
.space swFrameLength
kdiscLR:.space 4
.align 3 // ensure 8 byte alignment
kdiscFrameLength:
.align 6 // cache line alignment
SPECIAL_ENTRY_S(KiDispIntSwapContext,_TEXT$00)
mflr r.0 // get return address
lwz r.30, KiPcr+PcCurrentThread(r.0) // get current (old) thread
stwu r.sp, -kdiscFrameLength(r.sp) // buy stack frame
stw r.14, swFrame + ExGpr14(r.sp) // save gpr 14
stw r.26, swFrame + ExGpr26(r.sp) // save gprs 26 and 27
stw r.27, swFrame + ExGpr27(r.sp) //
// Gprs 28, 29, 30 and 31 saved/restored by KiDispatchInterrupt
stw r.0, kdiscLR(r.sp) // save return address
PROLOGUE_END(KiDispIntSwapContext)
stw r.31, PbCurrentThread(rPrcb) // set new thread current
// Reready current thread and swap context to the selected thread.
ori r.3, r.30, 0
#if !defined(NT_UP)
lwz r.27, [toc]KiContextSwapLock(r.2)
#endif
bl ..KiReadyThread // reready current thread
bl ..SwapContext // switch threads
// Restore registers and return.
lwz r.0, kdiscLR(r.sp) // restore return address
lwz r.26, swFrame + ExGpr26(r.sp) // restore gpr 26 and 27
mtlr r.0 // set return address
lwz r.27, swFrame + ExGpr27(r.sp) //
// Gprs 28, 29, 30 and 31 saved/restored by KiDispatchInterrupt
lwz r.14, swFrame + ExGpr14(r.sp) // restore gpr 14
addi r.sp, r.sp, kdiscFrameLength // return stack frame
SPECIAL_EXIT(KiDispIntSwapContext)
// VOID
// KiRequestSoftwareInterrupt (KIRQL RequestIrql)
// Routine Description:
// This function requests a software interrupt at the specified IRQL
// level.
// Arguments:
// RequestIrql (r.3) - Supplies the request IRQL value.
// Allowable values are APC_LEVEL (1)
// DPC_LEVEL (2)
// Return Value:
// None.
LEAF_ENTRY(KiRequestSoftwareInterrupt)
lbz r.6, KiPcr+PcCurrentIrql(r.0) // get current IRQL
rlwinm r.4, r.3, 31, 0x1 // transform 1 or 2 to 0 or 1
li r.5, 1 // non-zero value
cmpw r.6, r.3 // is current IRQL < requested IRQL?
stb r.5, KiPcr+PcSoftwareInterrupt(r.4) // set interrupt pending
blt ..KiDispatchSoftwareInterrupt // jump to dispatch interrupt if
// current IRQL low enough (note
// that this is a jump, not a call)
LEAF_EXIT(KiRequestSoftwareInterrupt)
Reference in New Issue View Git Blame Copy Permalink