599 lines
16 KiB
ArmAsm
599 lines
16 KiB
ArmAsm
// TITLE("Spin Locks")
|
||
//++
|
||
//
|
||
// Copyright (c) 1993 IBM Corporation
|
||
//
|
||
// Module Name:
|
||
//
|
||
// spinlock.s
|
||
//
|
||
// Abstract:
|
||
//
|
||
// This module implements the routines for acquiring and releasing
|
||
// spin locks.
|
||
//
|
||
// Author:
|
||
//
|
||
// Peter L. Johnston (plj@vnet.ibm.com) 28-Jun-93
|
||
//
|
||
// Environment:
|
||
//
|
||
// Kernel mode only.
|
||
//
|
||
// Revision History:
|
||
//
|
||
//--
|
||
|
||
#include "ksppc.h"
|
||
|
||
.extern ..KxReleaseSpinLock
|
||
|
||
|
||
SBTTL("Initialize Executive Spin Lock")
|
||
//++
|
||
//
|
||
// VOID
|
||
// KeInitializeSpinLock (
|
||
// IN PKSPIN_LOCK SpinLock
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function initialzies an executive spin lock.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// SpinLock (r3) - Supplies a pointer to a executive spinlock.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// None.
|
||
//
|
||
// Remarks:
|
||
//
|
||
// Equivalent C code for body of function.
|
||
// {
|
||
// *spinlock = 0;
|
||
// }
|
||
//
|
||
// Why is this in assembly code? I suspect simply so it is bundled
|
||
// with the rest of the spin lock code.
|
||
//--
|
||
|
||
LEAF_ENTRY(KeInitializeSpinLock)
|
||
|
||
li r.0, 0 // clear spin lock value by storing
|
||
stw r.0, 0(r.3) // zero at address from parameter.
|
||
|
||
LEAF_EXIT(KeInitializeSpinLock) // return
|
||
|
||
SBTTL("Acquire Executive Spin Lock")
|
||
//++
|
||
//
|
||
// VOID
|
||
// KeAcquireSpinLock (
|
||
// IN PKSPIN_LOCK SpinLock
|
||
// OUT PKIRQL OldIrql
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function raises the current IRQL to DISPATCH_LEVEL and acquires
|
||
// the specified executive spinlock.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// SpinLock (r3) - Supplies a pointer to an executive spinlock.
|
||
//
|
||
// OldIrql (r4) - Supplies a pointer to a variable that receives the
|
||
// the previous IRQL value.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// None.
|
||
//
|
||
// Remarks:
|
||
//
|
||
// In the UniProcessor version, we just raise IRQL, the lock address
|
||
// is never touched.
|
||
//
|
||
// In the MultiProcessor version the spinlock is taken after raising
|
||
// IRQL.
|
||
//
|
||
// N.B. The old IRQL must be stored AFTER the lock is acquired.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY_S(KeAcquireSpinLock,_TEXT$01)
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
DISABLE_INTERRUPTS(r.5, r.6)
|
||
|
||
lwz r.12, KiPcr+PcCurrentThread(r.0) // addr of current thread
|
||
|
||
ACQUIRE_SPIN_LOCK(r.3, r.12, r.11, kas.10, kas.15)
|
||
|
||
#endif
|
||
|
||
li r.0, DISPATCH_LEVEL // new IRQL = DISPATCH_LEVEL
|
||
lbz r.10,KiPcr+PcCurrentIrql(r.0) // get current IRQL
|
||
stb r.0, KiPcr+PcCurrentIrql(r.0) // set new IRQL
|
||
|
||
#if !defined(NT_UP)
|
||
ENABLE_INTERRUPTS(r.5)
|
||
#endif
|
||
|
||
stb r.10, 0(r.4) // return old IRQL
|
||
blr
|
||
|
||
#if !defined(NT_UP)
|
||
SPIN_ON_SPIN_LOCK_ENABLED(r.3, r.11, kas.10, kas.15, kas.17, r.5, r.6)
|
||
#endif
|
||
|
||
DUMMY_EXIT(KeAcquireSpinLock)
|
||
|
||
SBTTL("Acquire SpinLock and Raise to DPC")
|
||
//++
|
||
//
|
||
// KIRQL
|
||
// KeAcquireSpinLockRaiseToDpc (
|
||
// IN PKSPIN_LOCK SpinLock
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function raises the current IRQL to dispatcher level and acquires
|
||
// the specified spinlock.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// SpinLock (r3) - Supplies a pointer to the spinlock that is to be
|
||
// acquired.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// The previous IRQL is returned at the function value.
|
||
//
|
||
// Remarks:
|
||
//
|
||
// In the UniProcessor version, we just raise IRQL, the lock address
|
||
// is never touched.
|
||
//
|
||
// In the MultiProcessor version the spinlock is taken after raising
|
||
// IRQL.
|
||
//
|
||
// N.B. The old IRQL must be stored AFTER the lock is acquired.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY_S(KeAcquireSpinLockRaiseToDpc,_TEXT$01)
|
||
|
||
//
|
||
// On PPC, synchronization level is the same as dispatch level.
|
||
//
|
||
|
||
ALTERNATE_ENTRY(KeAcquireSpinLockRaiseToSynch)
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
DISABLE_INTERRUPTS(r.5, r.6)
|
||
|
||
lwz r.12, KiPcr+PcCurrentThread(r.0) // addr of current thread
|
||
|
||
ACQUIRE_SPIN_LOCK(r.3, r.12, r.11, kasrtd.10, kasrtd.15)
|
||
|
||
#endif
|
||
|
||
li r.0, DISPATCH_LEVEL // new IRQL = DISPATCH_LEVEL
|
||
lbz r.10,KiPcr+PcCurrentIrql(r.0) // get current IRQL
|
||
stb r.0, KiPcr+PcCurrentIrql(r.0) // set new IRQL
|
||
|
||
#if !defined(NT_UP)
|
||
ENABLE_INTERRUPTS(r.5)
|
||
#endif
|
||
|
||
ori r.3, r.10, 0 // return old IRQL
|
||
blr
|
||
|
||
#if !defined(NT_UP)
|
||
SPIN_ON_SPIN_LOCK_ENABLED(r.3, r.11, kasrtd.10, kasrtd.15, kasrtd.20, r.5, r.6)
|
||
#endif
|
||
|
||
DUMMY_EXIT(KeAcquireSpinLockRaiseToDpc)
|
||
|
||
SBTTL("Release Executive Spin Lock")
|
||
//++
|
||
//
|
||
// VOID
|
||
// KeReleaseSpinLock (
|
||
// IN PKSPIN_LOCK SpinLock
|
||
// IN KIRQL OldIrql
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function releases an executive spin lock and lowers the IRQL
|
||
// to its previous value.
|
||
//
|
||
// N.B. This routine is entered at DISPATCH_LEVEL.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// SpinLock (r.3) - Supplies a pointer to an executive spin lock.
|
||
//
|
||
// OldIrql (r.4) - Supplies the previous IRQL value.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// None.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY_S(KeReleaseSpinLock, _TEXT$01)
|
||
|
||
cmpwi r.4, DISPATCH_LEVEL // check if new IRQL < DISPATCH
|
||
|
||
#if !defined(NT_UP)
|
||
li r.0, 0
|
||
RELEASE_SPIN_LOCK(r.3, r.0)
|
||
#endif
|
||
|
||
bgelr // if target IRQL >= DISPATCH
|
||
// just return
|
||
DISABLE_INTERRUPTS(r.7,r.10)
|
||
lhz r.10, KiPcr+PcSoftwareInterrupt(r.0) // s/w interrupt pending?
|
||
stb r.4, KiPcr+PcCurrentIrql(r.0) // set target IRQL
|
||
cmpw cr.6, r.10, r.4
|
||
srwi. r.5, r.10, 8 // isolate DPC pending
|
||
cmpwi cr.7, r.4, APC_LEVEL // compare IRQL to APC_LEVEL
|
||
|
||
//
|
||
// Possible values for SoftwareInterrupt (r.10) are
|
||
//
|
||
// 0x0101 DPC and APC interrupt pending
|
||
// 0x0100 DPC interrupt pending
|
||
// 0x0001 APC interrupt pending
|
||
// 0x0000 No software interrupt pending (unlikely but possible)
|
||
//
|
||
// Possible values for current IRQL are zero or one. By comparing
|
||
// SoftwareInterrupt against current IQRL (above) we can quickly see
|
||
// if any software interrupts are valid at this time.
|
||
//
|
||
// Calculate correct IRQL for the interrupt we are processing. If DPC
|
||
// then we need to be at DISPATCH_LEVEL which is one greater than APC_
|
||
// LEVEL. r.5 contains one if we are going to run a DPC, so we add
|
||
// APC_LEVEL to r.5 to get the desired IRQL.
|
||
//
|
||
|
||
addi r.4, r.5, APC_LEVEL // calculate new IRQL
|
||
|
||
ble cr.6,Enable // jif no valid interrupt
|
||
|
||
//
|
||
// A software interrupt is pending and the new IRQL allows it to be
|
||
// taken at this time. Branch directly to KxReleaseSpinLock (ctxswap.s)
|
||
// to dispatch the interrupt. KxReleaseSpinLock returns directly to
|
||
// KeReleaseSpinLock's caller.
|
||
//
|
||
// WARNING: KxReleaseSpinLock is dependent on the values in r.4, r.7, r.12
|
||
// and condition register fields cr.0 and cr.7.
|
||
//
|
||
|
||
b ..KxReleaseSpinLock
|
||
|
||
Enable: ENABLE_INTERRUPTS(r.7)
|
||
|
||
LEAF_EXIT(KeReleaseSpinLock) // return
|
||
|
||
SBTTL("Try To Acquire Executive Spin Lock")
|
||
//++
|
||
//
|
||
// BOOLEAN
|
||
// KeTryToAcquireSpinLock (
|
||
// IN PKSPIN_LOCK SpinLock
|
||
// OUT PKIRQL OldIrql
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function raises the current IRQL to DISPATCH_LEVEL and attempts
|
||
// to acquires the specified executive spinlock. If the spinlock can be
|
||
// acquired, then TRUE is returned. Otherwise, the IRQL is restored to
|
||
// its previous value and FALSE is returned.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// SpinLock (r.3) - Supplies a pointer to a executive spinlock.
|
||
//
|
||
// OldIrql (r.4) - Supplies a pointer to a variable that receives the
|
||
// the previous IRQL value.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// If the spin lock is acquired, then a value of TRUE is returned.
|
||
// Otherwise, a value of FALSE is returned. UP systems always succeed.
|
||
// On MP systems we test if the lock is taken with interrupts disabled,
|
||
// so that we don't have to check for any DPCs/APCs that could be queued
|
||
// while we have priority at dispatch level.
|
||
//
|
||
// N.B. The old IRQL must be stored AFTER the lock is acquired.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY_S(KeTryToAcquireSpinLock,_TEXT$01)
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
//
|
||
// Try to acquire the specified spinlock.
|
||
//
|
||
|
||
DISABLE_INTERRUPTS(r.9, r.8)
|
||
|
||
lwz r.12, KiPcr+PcCurrentThread(r.0) // addr of current thread
|
||
|
||
TRY_TO_ACQUIRE_SPIN_LOCK(r.3, r.12, r.11, ktas.10, ktas.20)
|
||
|
||
#endif
|
||
|
||
//
|
||
// Raise IRQL and indicate success.
|
||
//
|
||
|
||
lbz r.10,KiPcr+PcCurrentIrql(r.0) // get current IRQL
|
||
li r.0, DISPATCH_LEVEL // new IRQL = DISPATCH_LEVEL
|
||
li r.3, TRUE // set return value (success)
|
||
stb r.10, 0(r.4) // return old IRQL
|
||
stb r.0, KiPcr+PcCurrentIrql(r.0) // set new IRQL
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
ENABLE_INTERRUPTS(r.9)
|
||
|
||
blr // return
|
||
|
||
//
|
||
// The attempt to acquire the specified spin lock failed. Indicate failure.
|
||
//
|
||
|
||
ktas.20:
|
||
ENABLE_INTERRUPTS(r.9)
|
||
|
||
li r.3, FALSE // set return value (failure)
|
||
|
||
#endif
|
||
|
||
LEAF_EXIT(KeTryToAcquireSpinLock)
|
||
|
||
SBTTL("Acquire Kernel Spin Lock")
|
||
//++
|
||
//
|
||
// KIRQL
|
||
// KiAcquireSpinLock (
|
||
// IN PKSPIN_LOCK SpinLock
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function acquires a kernel spin lock.
|
||
//
|
||
// N.B. This function assumes that the current IRQL is set properly.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// SpinLock (a0) - Supplies a pointer to a kernel spin lock.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// None.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY_S(KiAcquireSpinLock,_TEXT$01)
|
||
|
||
ALTERNATE_ENTRY(KeAcquireSpinLockAtDpcLevel)
|
||
|
||
#if !defined(NT_UP)
|
||
lwz r.12, KiPcr+PcCurrentThread(r.0) // addr of current thread
|
||
|
||
ACQUIRE_SPIN_LOCK(r.3, r.12, r.11, kas.20, kas.30)
|
||
#endif
|
||
|
||
blr
|
||
|
||
#if !defined(NT_UP)
|
||
SPIN_ON_SPIN_LOCK(r.3, r.11, kas.20, kas.30)
|
||
#endif
|
||
|
||
DUMMY_EXIT(KiAcquireSpinLock)
|
||
|
||
SBTTL("Release Kernel Spin Lock")
|
||
//++
|
||
//
|
||
// VOID
|
||
// KiReleaseSpinLock (
|
||
// IN PKSPIN_LOCK SpinLock
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function releases a kernel spin lock.
|
||
//
|
||
// N.B. This function assumes that the current IRQL is set properly.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// SpinLock (a0) - Supplies a pointer to an executive spin lock.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// None.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY_S(KiReleaseSpinLock,_TEXT$01)
|
||
|
||
ALTERNATE_ENTRY(KeReleaseSpinLockFromDpcLevel)
|
||
|
||
#if !defined(NT_UP)
|
||
li r.0, 0
|
||
RELEASE_SPIN_LOCK(r.3, r.0)
|
||
#endif
|
||
|
||
LEAF_EXIT(KiReleaseSpinLock) // return
|
||
|
||
SBTTL("Try To Acquire Kernel Spin Lock")
|
||
//++
|
||
//
|
||
// KIRQL
|
||
// KiTryToAcquireSpinLock (
|
||
// IN PKSPIN_LOCK SpinLock
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function attempts to acquires the specified kernel spinlock. If
|
||
// the spinlock can be acquired, then TRUE is returned. Otherwise, FALSE
|
||
// is returned.
|
||
//
|
||
// N.B. This function assumes that the current IRQL is set properly.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// SpinLock (r.3) - Supplies a pointer to a kernel spin lock.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// If the spin lock is acquired, then a value of TRUE is returned.
|
||
// Otherwise, a value of FALSE is returned.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY_S(KiTryToAcquireSpinLock,_TEXT$01)
|
||
|
||
#if defined(NT_UP)
|
||
|
||
li r.3, TRUE // set return value (success)
|
||
|
||
#else
|
||
|
||
lwz r.12, KiPcr+PcCurrentThread(r.0) // addr of current thread
|
||
|
||
TRY_TO_ACQUIRE_SPIN_LOCK(r.3, r.12, r.11, kitas.10, kitas.20)
|
||
li r.3, TRUE // set return value (success)
|
||
|
||
blr // return
|
||
|
||
kitas.20:
|
||
li r.3, FALSE // set return value (failure)
|
||
#endif
|
||
LEAF_EXIT(KiTryToAcquireSpinLock)
|
||
|
||
|
||
#if !defined(NT_UP)
|
||
#if SPINDBG
|
||
.struct 0
|
||
.space StackFrameHeaderLength
|
||
kasdR5: .space 4 // saved R5
|
||
kasdR6: .space 4 // saved R6
|
||
kasdR7: .space 4 // saved R7
|
||
kasdR8: .space 4 // saved R8
|
||
.align 3 // 8 byte align
|
||
kasdFrameLength:
|
||
|
||
SPECIAL_ENTRY_S(KiAcquireSpinLockDbg,_TEXT$01)
|
||
stwu sp,-kasdFrameLength(sp)
|
||
stw r5,kasdR5(sp) // save r5
|
||
stw r6,kasdR6(sp) // save r6
|
||
stw r7,kasdR7(sp) // save r7
|
||
stw r8,kasdR8(sp) // save r8
|
||
PROLOGUE_END(KiAcquireSpinLockDbg)
|
||
lwz r5,[toc]KiSpinLockLimit(r.toc)
|
||
lwz r5,0(r5)
|
||
mflr r7
|
||
CHKBRK(r6,kasd.05)
|
||
CHKLOCK(r6,r3,kasd.10)
|
||
DBGSTORE_IRR(r6,r8,0x7711,r4,r7)
|
||
kasd.10:
|
||
lwarx r6,0,r3 // load word at (r.3) and reserve
|
||
cmpwi r6,0 // check if locked
|
||
bne- kasd.20 // jif locked (predict NOT taken)
|
||
stwcx. r4,0,r3 // set lock owned
|
||
bne- kasd.20 // try again if store failed
|
||
isync // allow no readahead
|
||
CHKLOCK(r6,r3,kasd.11)
|
||
DBGSTORE_IRR(r6,r8,0x7712,r4,r7)
|
||
kasd.11:
|
||
lwz r8,kasdR8(sp) // restore r8
|
||
lwz r7,kasdR7(sp) // restore r7
|
||
lwz r6,kasdR6(sp) // restore r6
|
||
lwz r5,kasdR5(sp) // restore r5
|
||
addi sp,sp,kasdFrameLength // deallocate stack frame
|
||
blr // return
|
||
kasd.20:
|
||
subi r5,r5,1
|
||
cmpwi r5,0
|
||
beq- kasd.30
|
||
kasd.25:
|
||
lwz r6,0(r3)
|
||
cmpwi r6,0
|
||
beq+ kasd.10
|
||
b kasd.20
|
||
kasd.30:
|
||
//DBGSTORE_IRRR(r6,r8,0x7713,r3,r4,r7)
|
||
twi 31,0,0x16
|
||
lwz r5,[toc]KiSpinLockLimit(r.toc)
|
||
lwz r5,0(r5)
|
||
b kasd.25
|
||
DUMMY_EXIT(KiAcquireSpinLockDbg)
|
||
|
||
SPECIAL_ENTRY_S(KiTryToAcquireSpinLockDbg,_TEXT$01)
|
||
stwu sp,-kasdFrameLength(sp)
|
||
stw r5,kasdR5(sp) // save r5
|
||
stw r6,kasdR6(sp) // save r6
|
||
stw r7,kasdR7(sp) // save r7
|
||
stw r8,kasdR8(sp) // save r8
|
||
PROLOGUE_END(KiTryToAcquireSpinLockDbg)
|
||
lwz r5,[toc]KiSpinLockLimit(r.toc)
|
||
lwz r5,0(r5)
|
||
ori r6,r3,0
|
||
mflr r7
|
||
CHKBRK(r3,ktasd.05)
|
||
CHKLOCK(r3,r6,ktasd.10)
|
||
DBGSTORE_IRR(r3,r8,0x7721,r4,r7)
|
||
ktasd.10:
|
||
lwarx r3,0,r6 // load word at (r.3) and reserve
|
||
cmpwi r3,0 // check if locked
|
||
bne- ktasd.40 // jif locked (predict NOT taken)
|
||
stwcx. r4,0,r6 // set lock owned
|
||
bne- ktasd.20 // try again if store failed
|
||
isync // allow no readahead
|
||
li r3,TRUE
|
||
ktasd.15:
|
||
CHKLOCK(r8,r6,ktasd.16)
|
||
DBGSTORE_IRR(r6,r8,0x7722,r4,r7)
|
||
ktasd.16:
|
||
lwz r8,kasdR8(sp) // restore r8
|
||
lwz r7,kasdR7(sp) // restore r7
|
||
lwz r6,kasdR6(sp) // restore r6
|
||
lwz r5,kasdR5(sp) // restore r5
|
||
addi sp,sp,kasdFrameLength // deallocate stack frame
|
||
blr // return
|
||
ktasd.20:
|
||
subi r5,r5,1
|
||
cmpwi r5,0
|
||
bne+ ktasd.10
|
||
ktasd.30:
|
||
//DBGSTORE_IRRR(r3,r8,0x7723,r3,r4,r7)
|
||
twi 31,0,0x16
|
||
lwz r5,[toc]KiSpinLockLimit(r.toc)
|
||
lwz r5,0(r5)
|
||
b ktasd.10
|
||
ktasd.40:
|
||
li r3,FALSE
|
||
b ktasd.15
|
||
DUMMY_EXIT(KiTryToAcquireSpinLockDbg)
|
||
#endif
|
||
#endif
|