1220 lines
36 KiB
ArmAsm
1220 lines
36 KiB
ArmAsm
// TITLE("Interlocked Support")
|
||
//++
|
||
//
|
||
// Copyright (c) 1993 IBM Corporation
|
||
//
|
||
// Module Name:
|
||
//
|
||
// intrlock.s
|
||
//
|
||
// Abstract:
|
||
//
|
||
// This module implements functions to support interlocked operations.
|
||
// Interlocked operations can only operate on nonpaged data and the
|
||
// specified spinlock cannot be used for any other purpose.
|
||
//
|
||
// Author:
|
||
//
|
||
// Chuck Bauman 3-Sep-1993
|
||
//
|
||
// Environment:
|
||
//
|
||
// Kernel mode.
|
||
//
|
||
// Revision History:
|
||
//
|
||
//--
|
||
|
||
#include "ksppc.h"
|
||
|
||
#if COLLECT_PAGING_DATA
|
||
.extern KeNumberProcessors
|
||
.extern KiProcessorBlock
|
||
.extern ..RtlCopyMemory
|
||
#endif
|
||
|
||
|
||
// SBTTL("Interlocked Add Large Integer")
|
||
//++
|
||
//
|
||
// LARGE_INTEGER
|
||
// ExInterlockedAddLargeInteger (
|
||
// IN PLARGE_INTEGER Addend,
|
||
// IN LARGE_INTEGER Increment,
|
||
// IN PKSPIN_LOCK Lock
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function performs an interlocked add of an increment value to an
|
||
// addend variable of type large integer. The initial value of the addend
|
||
// variable is returned as the function value.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// Addend (r4) - Supplies a pointer to a variable whose value is to be
|
||
// adjusted by the increment value.
|
||
//
|
||
// Increment (r5, r6) - Supplies the increment value to be added to the
|
||
// addend variable.
|
||
//
|
||
// Lock (r7) - Supplies a pointer to a spin lock to be used to
|
||
// synchronize access to the addend variable.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// The initial value of the addend variable is stored at the address
|
||
// supplied by r3.
|
||
//
|
||
// Implementation Note:
|
||
//
|
||
// The arithmetic for this function is performed as if this were an
|
||
// unsigned large integer since this routine may not incur an overflow
|
||
// exception.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY(ExInterlockedAddLargeInteger)
|
||
|
||
//
|
||
// N.B. ExInterlockedExchangeAddLargeInteger is the same as
|
||
// ExInterlockedAddLargeInteger on 32-bit machines.
|
||
// On 64-bit machines, an optimized version of the Exchange
|
||
// version could be implemented using ldarx and stdcx.
|
||
// The optimized version wouldn't need to take the spin lock.
|
||
//
|
||
|
||
ALTERNATE_ENTRY(ExInterlockedExchangeAddLargeInteger)
|
||
|
||
DISABLE_INTERRUPTS(r.8,r.12) // disable interrupts
|
||
// r.8 <- previous msr value
|
||
// r.12 <- new (disabled) msr
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
ACQUIRE_SPIN_LOCK(r.7, r.7, r.9, addli, addliw)
|
||
#endif
|
||
|
||
lwz r.10,0(r.4) // get low part of addend value
|
||
lwz r.11,4(r.4) // get high part of addend value
|
||
addc r.5,r.10,r.5 // add low parts of large integer,(CA?)
|
||
adde r.6,r.11,r.6 // add high parts of large integer + CA
|
||
stw r.5,0(r.4) // store low part of result
|
||
stw r.6,4(r.4) // store high part of result
|
||
|
||
#if !defined(NT_UP)
|
||
RELEASE_SPIN_LOCK(r.7, r.9)
|
||
#endif
|
||
|
||
ENABLE_INTERRUPTS(r.8) // enable interrupts
|
||
|
||
stw r.10,0(r.3) // set low part of initial value
|
||
stw r.11,4(r.3) // set high part of initial value
|
||
|
||
blr // return
|
||
|
||
#if !defined(NT_UP)
|
||
SPIN_ON_SPIN_LOCK_ENABLED(r.7, r.9, addli, addliw, addlix, r.8, r.12)
|
||
#endif
|
||
|
||
DUMMY_EXIT(ExInterlockedAddLargeInteger)
|
||
|
||
// SBTTL("Interlocked Add Large Statistic")
|
||
//++
|
||
//
|
||
// VOID
|
||
// ExInterlockedAddLargeStatistic (
|
||
// IN PLARGE_INTEGER Addend,
|
||
// IN ULONG Increment
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function performs an interlocked add of an increment value to an
|
||
// addend variable of type large integer.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// Addend (r.3) - Supplies a pointer to a variable whose value is to be
|
||
// adjusted by the increment value.
|
||
//
|
||
// Increment (r.4) - Supplies the increment value to be added to the
|
||
// addend variable.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// None.
|
||
//
|
||
// Implementation Note:
|
||
//
|
||
// The arithmetic for this function is performed as if this were an
|
||
// unsigned large integer since this routine may not incur an overflow
|
||
// exception.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY(ExInterlockedAddLargeStatistic)
|
||
|
||
storelsfailed:
|
||
lwarx r.8, 0, r.3 // get low part of large statistic
|
||
addc r.9, r.8, r.4 // add increment to low part
|
||
stwcx. r.9, 0, r.3 // store result of low part add
|
||
bne- storelsfailed // if ne, store conditional failed
|
||
|
||
subfe. r.0, r.0, r.0 // check carry bit (result is 0
|
||
// if CA is set)
|
||
bnelr+ // if ne, carry clear, so return
|
||
|
||
li r.10, 4 // high part offset
|
||
|
||
storels2failed:
|
||
lwarx r.8, r.10, r.3 // get high part of large statistic
|
||
addi r.8, r.8, 1 // add carry to high part
|
||
stwcx. r.8, r.10, r.3 // store result of high part add
|
||
bne- storels2failed // if ne, store conditional failed
|
||
|
||
LEAF_EXIT(ExInterlockedAddLargeStatistic)
|
||
|
||
// SBTTL("Interlocked Add Unsigned Long")
|
||
//++
|
||
//
|
||
// ULONG
|
||
// ExInterlockedAddUlong (
|
||
// IN PULONG Addend,
|
||
// IN ULONG Increment,
|
||
// IN PKSPIN_LOCK Lock
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function performs an interlocked add of an increment value to an
|
||
// addend variable of type unsigned long. The initial value of the addend
|
||
// variable is returned as the function value.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// Addend (r.3) - Supplies a pointer to a variable whose value is to be
|
||
// adjusted by the increment value.
|
||
//
|
||
// Increment (r.4) - Supplies the increment value to be added to the
|
||
// addend variable.
|
||
//
|
||
// Lock (r.5) - Supplies a pointer to a spin lock to be used to synchronize
|
||
// access to the addend variable.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// The initial value of the addend variable.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY(ExInterlockedAddUlong)
|
||
|
||
ori r.6,r.3,0 // move addend address
|
||
|
||
DISABLE_INTERRUPTS(r.8,r.12) // disable interrupts
|
||
// r.8 <- previous msr value
|
||
// r.12 <- new (disabled) msr
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
ACQUIRE_SPIN_LOCK(r.5, r.5, r.9, addul, addulw)
|
||
#endif
|
||
|
||
lwz r.3,0(r.6) // get initial addend value
|
||
add r.4,r.3,r.4 // compute adjusted value
|
||
stw r.4,0(r.6) // set updated addend value
|
||
|
||
#if !defined(NT_UP)
|
||
RELEASE_SPIN_LOCK(r.5, r.9)
|
||
#endif
|
||
|
||
|
||
ENABLE_INTERRUPTS(r.8) // enable interrupts
|
||
|
||
blr // return
|
||
|
||
#if !defined(NT_UP)
|
||
SPIN_ON_SPIN_LOCK_ENABLED(r.5, r.9, addul, addulw, addulx, r.8, r.12)
|
||
#endif
|
||
|
||
DUMMY_EXIT(ExInterlockedAddUlong)
|
||
|
||
// SBTTL("Interlocked Exchange Unsigned Long")
|
||
//++
|
||
//
|
||
// ULONG
|
||
// ExInterlockedExchangeUlong (
|
||
// IN PULONG Source,
|
||
// IN ULONG Value,
|
||
// IN PKSPIN_LOCK Lock
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function performs an interlocked exchange of a longword value with
|
||
// a longword in memory and returns the memory value.
|
||
//
|
||
// N.B. There is an alternate entry point provided for this routine which
|
||
// is PPC target specific and whose prototype does not include the
|
||
// spinlock parameter. Since the routine never refers to the spinlock
|
||
// parameter, no additional code is required.
|
||
//
|
||
// There is an additional alternate entry point, InterlockedExchange,
|
||
// which is not platform-specific and which also does not take the
|
||
// spinlock parameter.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// Source (r.3) - Supplies a pointer to a variable whose value is to be
|
||
// exchanged.
|
||
//
|
||
// Value (r.4) - Supplies the value to exchange with the source value.
|
||
//
|
||
// Lock (r.5) - Supplies a pointer to a spin lock to be used to synchronize
|
||
// access to the source variable.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// The source value is returned as the function value.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY(ExInterlockedExchangeUlong)
|
||
|
||
ALTERNATE_ENTRY(ExPpcInterlockedExchangeUlong)
|
||
ALTERNATE_ENTRY(InterlockedExchange)
|
||
|
||
exchgfailed:
|
||
lwarx r.5,0,r.3 // get current source value
|
||
stwcx. r.4,0,r.3 // set new source value
|
||
bne- exchgfailed // if ne, store conditional failed
|
||
|
||
ori r.3,r.5,0 // return old value
|
||
|
||
LEAF_EXIT(ExInterlockedExchangeUlong) // return
|
||
|
||
// SBTTL("Interlocked Decrement Long")
|
||
//++
|
||
//
|
||
// INTERLOCKED_RESULT
|
||
// ExInterlockedDecrementLong (
|
||
// IN PLONG Addend,
|
||
// IN PKSPIN_LOCK Lock
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function performs an interlocked decrement on an addend variable
|
||
// of type signed long. The sign and whether the result is zero is returned
|
||
// as the function value.
|
||
//
|
||
// N.B. There is an alternate entry point provided for this routine which
|
||
// is PPC target specific and whose prototype does not include the
|
||
// spinlock parameter. Since the routine never refers to the spinlock
|
||
// parameter, no additional code is required.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// Addend (r.3) - Supplies a pointer to a variable whose value is to be
|
||
// decremented.
|
||
//
|
||
// Lock (r.4) - Supplies a pointer to a spin lock to be used to synchronize
|
||
// access to the addend variable.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// RESULT_NEGATIVE is returned if the resultant addend value is negative.
|
||
// RESULT_ZERO is returned if the resultant addend value is zero.
|
||
// RESULT_POSITIVE is returned if the resultant addend value is positive.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY(ExInterlockedDecrementLong)
|
||
|
||
ALTERNATE_ENTRY(ExPpcInterlockedDecrementLong)
|
||
|
||
decfailed:
|
||
lwarx r.5,0,r.3 // get current addend value
|
||
subi r.4,r.5,1 // decrement addend value
|
||
stwcx. r.4,0,r.3 // set new addend value
|
||
bne- decfailed // if ne, store conditional failed
|
||
|
||
addic. r.5,r.5,-1 // Reset CR0
|
||
mfcr r.3 // Rotate bit 0 & 1 and right justify
|
||
rlwinm r.3,r.3,2,30,31 // 0 = 0, 1 = positive, 2 = negative
|
||
neg r.3,r.3 // 0 = 0, -1 = positive, -2 = negative
|
||
|
||
LEAF_EXIT(ExInterlockedDecrementLong) // return
|
||
|
||
// SBTTL("Interlocked Decrement")
|
||
//++
|
||
//
|
||
// LONG
|
||
// InterlockedDecrement (
|
||
// IN PLONG Addend
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function performs an interlocked decrement on an addend variable
|
||
// of type signed long. The result is returned as the function value.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// Addend (r.3) - Supplies a pointer to a variable whose value is to be
|
||
// decremented.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// (r.3) The result of the decrement is returned.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY(InterlockedDecrement)
|
||
|
||
dec2failed:
|
||
lwarx r.5,0,r.3 // get current addend value
|
||
subi r.4,r.5,1 // decrement addend value
|
||
stwcx. r.4,0,r.3 // set new addend value
|
||
bne- dec2failed // if ne, store conditional failed
|
||
|
||
ori r.3,r.4,0 // return result
|
||
|
||
LEAF_EXIT(InterlockedDecrement) // return
|
||
|
||
// SBTTL("Interlocked Increment Long")
|
||
//++
|
||
//
|
||
// INTERLOCKED_RESULT
|
||
// ExInterlockedIncrementLong (
|
||
// IN PLONG Addend,
|
||
// IN PKSPIN_LOCK Lock
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function performs an interlocked increment on an addend variable
|
||
// of type signed long. The sign and whether the result is zero is returned
|
||
// as the function value.
|
||
//
|
||
// N.B. There is an alternate entry point provided for this routine which
|
||
// is PPC target specific and whose prototype does not include the
|
||
// spinlock parameter. Since the routine never refers to the spinlock
|
||
// parameter, no additional code is required.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// Addend (r.3) - Supplies a pointer to a variable whose value is to be
|
||
// incremented.
|
||
//
|
||
// Lock (r.4) - Supplies a pointer to a spin lock to be used to synchronize
|
||
// access to the addend variable.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// RESULT_NEGATIVE is returned if the resultant addend value is negative.
|
||
// RESULT_ZERO is returned if the resultant addend value is zero.
|
||
// RESULT_POSITIVE is returned if the resultant addend value is positive.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY(ExInterlockedIncrementLong)
|
||
|
||
ALTERNATE_ENTRY(ExPpcInterlockedIncrementLong)
|
||
|
||
incfailed:
|
||
lwarx r.5,0,r.3 // get current addend value
|
||
addi r.4,r.5,1 // increment addend value
|
||
stwcx. r.4,0,r.3 // set new addend value
|
||
bne- incfailed // if ne, store conditional failed
|
||
|
||
addic. r.5,r.5,1 // Reset CR0
|
||
mfcr r.3 // Rotate bit 0 & 1 and right justify
|
||
rlwinm r.3,r.3,2,30,31 // 0 = 0, 1 = positive, 2 = negative
|
||
neg r.3,r.3 // 0 = 0, -1 = positive, -2 = negative
|
||
|
||
LEAF_EXIT(ExInterlockedIncrementLong) // return
|
||
|
||
// SBTTL("Interlocked Increment")
|
||
//++
|
||
//
|
||
// LONG
|
||
// InterlockedIncrement (
|
||
// IN PLONG Addend
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function performs an interlocked increment on an addend variable
|
||
// of type signed long. The result is returned as the function value.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// Addend (r.3) - Supplies a pointer to a variable whose value is to be
|
||
// incremented.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// (r.3) The result of the increment is returned.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY(InterlockedIncrement)
|
||
|
||
inc2failed:
|
||
lwarx r.5,0,r.3 // get current addend value
|
||
addi r.4,r.5,1 // increment addend value
|
||
stwcx. r.4,0,r.3 // set new addend value
|
||
bne- inc2failed // if ne, store conditional failed
|
||
|
||
ori r.3,r.4,0 // return result
|
||
|
||
LEAF_EXIT(InterlockedIncrement) // return
|
||
|
||
// SBTTL("Interlocked Compare Exchange")
|
||
//++
|
||
//
|
||
// PVOID
|
||
// InterlockedCompareExchange (
|
||
// IN OUT PVOID *Destination,
|
||
// IN PVOID Exchange,
|
||
// IN PVOID Comperand
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function performs an interlocked compare of the destination
|
||
// value with the comperand value. If the destination value is equal
|
||
// to the comperand value, then the exchange value is stored in the
|
||
// destination. Otherwise, no opeation is performed.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// Destination (r.3) - Supplies a pointer to the destination value.
|
||
//
|
||
// Exchange (r.4) - Supplies the exchange.
|
||
//
|
||
// Comperand (r.5) - Supplies the comperand value.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// The initial destination value is returned as the function value.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY(InterlockedCompareExchange)
|
||
|
||
Icmp10: lwarx r.6,0,r.3 // get current operand value
|
||
cmpw r.6,r.5 // compare with comperand
|
||
bne- Icmp20 // if ne, compare failed
|
||
stwcx. r.4,0,r.3 // set new operand value
|
||
bne- Icmp10 // if ne, store conditional failed
|
||
|
||
Icmp20: ori r.3,r.6,0 // return result
|
||
|
||
LEAF_EXIT(InterlockedCompareExchange) // return
|
||
|
||
// SBTTL("Interlocked Exchange Add")
|
||
//++
|
||
//
|
||
// LONG
|
||
// InterlockedExchangeAdd (
|
||
// IN PLONG Addend,
|
||
// IN LONG Increment
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function performs an interlocked add of an increment value to an
|
||
// addend variable of type unsigned long. The initial value of the addend
|
||
// variable is returned as the function value.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// Addend (r.3) - Supplies a pointer to a variable whose value is to be
|
||
// adjusted by the increment value.
|
||
//
|
||
// Increment (r.4) - Supplies the increment value to be added to the
|
||
// addend variable.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// The initial value of the addend variable.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY(InterlockedExchangeAdd)
|
||
|
||
add1failed:
|
||
lwarx r.5,0,r.3 // get current addend value
|
||
add r.6,r.5,r.4 // increment addend value
|
||
stwcx. r.6,0,r.3 // set new addend value
|
||
bne- add1failed // if ne, store conditional failed
|
||
|
||
ori r.3,r.5,0 // return result
|
||
|
||
LEAF_EXIT(InterlockedExchangeAdd) // return
|
||
|
||
// SBTTL("Interlocked Insert Head List")
|
||
//++
|
||
//
|
||
// PLIST_ENTRY
|
||
// ExInterlockedInsertHeadList (
|
||
// IN PLIST_ENTRY ListHead,
|
||
// IN PLIST_ENTRY ListEntry,
|
||
// IN PKSPIN_LOCK Lock
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function inserts an entry at the head of a doubly linked list
|
||
// so that access to the list is synchronized in a multiprocessor system.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// ListHead (r.3) - Supplies a pointer to the head of the doubly linked
|
||
// list into which an entry is to be inserted.
|
||
//
|
||
// ListEntry (r.4) - Supplies a pointer to the entry to be inserted at the
|
||
// head of the list.
|
||
//
|
||
// Lock (r.5) - Supplies a pointer to a spin lock to be used to synchronize
|
||
// access to the list.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// Pointer to entry that was at the head of the list or NULL if the list
|
||
// was empty.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY(ExInterlockedInsertHeadList)
|
||
|
||
ori r.6,r.3,0 // move listhead address
|
||
|
||
DISABLE_INTERRUPTS(r.8,r.12) // disable interrupts
|
||
// r.8 <- previous msr value
|
||
// r.12 <- new (disabled) msr
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
ACQUIRE_SPIN_LOCK(r.5, r.5, r.9, inshl, inshlw)
|
||
#endif
|
||
|
||
lwz r.7,LsFlink(r.6) // get address of next entry
|
||
stw r.6,LsBlink(r.4) // store previous link in entry
|
||
stw r.7,LsFlink(r.4) // store next link in entry
|
||
xor. r.3,r.7,r.6 // check if list was empty
|
||
stw r.4,LsBlink(r.7) // store previous link in next
|
||
stw r.4,LsFlink(r.6) // store next link in head
|
||
beq nullhlist // if eq, list was null
|
||
ori r.3,r.7,0 // return previous entry at head
|
||
nullhlist:
|
||
|
||
#if !defined(NT_UP)
|
||
RELEASE_SPIN_LOCK(r.5, r.9)
|
||
#endif
|
||
|
||
ENABLE_INTERRUPTS(r.8) // enable interrupts
|
||
|
||
blr // return
|
||
|
||
#if !defined(NT_UP)
|
||
SPIN_ON_SPIN_LOCK_ENABLED(r.5, r.9, inshl, inshlw, inshlx, r.8, r.12)
|
||
#endif
|
||
|
||
DUMMY_EXIT(ExInterlockedInsertHeadList)
|
||
|
||
|
||
// SBTTL("Interlocked Insert Tail List")
|
||
//++
|
||
//
|
||
// PLIST_ENTRY
|
||
// ExInterlockedInsertTailList (
|
||
// IN PLIST_ENTRY ListHead,
|
||
// IN PLIST_ENTRY ListEntry,
|
||
// IN PKSPIN_LOCK Lock
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function inserts an entry at the tail of a doubly linked list
|
||
// so that access to the list is synchronized in a multiprocessor system.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// ListHead (r.3) - Supplies a pointer to the head of the doubly linked
|
||
// list into which an entry is to be inserted.
|
||
//
|
||
// ListEntry (r.4) - Supplies a pointer to the entry to be inserted at the
|
||
// tail of the list.
|
||
//
|
||
// Lock (r.5) - Supplies a pointer to a spin lock to be used to synchronize
|
||
// access to the list.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// Pointer to entry that was at the tail of the list or NULL if the list
|
||
// was empty.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY(ExInterlockedInsertTailList)
|
||
|
||
ori r.6,r.3,0 // move listhead address
|
||
|
||
DISABLE_INTERRUPTS(r.8,r.12) // disable interrupts
|
||
// r.8 <- previous msr value
|
||
// r.12 <- new (disabled) msr
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
ACQUIRE_SPIN_LOCK(r.5, r.5, r.9, instl, instlw)
|
||
#endif
|
||
|
||
lwz r.7,LsBlink(r.6) // get address of previous entry
|
||
stw r.6,LsFlink(r.4) // store next link in entry
|
||
stw r.7,LsBlink(r.4) // store previous link in entry
|
||
xor. r.3,r.7,r.6 // check if list was empty
|
||
stw r.4,LsBlink(r.6) // store previous link in next
|
||
stw r.4,LsFlink(r.7) // store next link in head
|
||
beq nulltlist // if eq, list was empty
|
||
ori r.3,r.7,0 // return previous entry at tail
|
||
nulltlist:
|
||
|
||
#if !defined(NT_UP)
|
||
RELEASE_SPIN_LOCK(r.5, r.9)
|
||
#endif
|
||
|
||
ENABLE_INTERRUPTS(r.8) // enable interrupts
|
||
|
||
blr // return
|
||
|
||
#if !defined(NT_UP)
|
||
SPIN_ON_SPIN_LOCK_ENABLED(r.5, r.9, instl, instlw, instlx, r.8, r.12)
|
||
#endif
|
||
|
||
DUMMY_EXIT(ExInterlockedInsertTailList)
|
||
|
||
// SBTTL("Interlocked Remove Head List")
|
||
//++
|
||
//
|
||
// PLIST_ENTRY
|
||
// ExInterlockedRemoveHeadList (
|
||
// IN PLIST_ENTRY ListHead,
|
||
// IN PKSPIN_LOCK Lock
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function removes an entry from the head of a doubly linked list
|
||
// so that access to the list is synchronized in a multiprocessor system.
|
||
// If there are no entries in the list, then a value of NULL is returned.
|
||
// Otherwise, the address of the entry that is removed is returned as the
|
||
// function value.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// ListHead (r.3) - Supplies a pointer to the head of the doubly linked
|
||
// list from which an entry is to be removed.
|
||
//
|
||
// Lock (r.4) - Supplies a pointer to a spin lock to be used to synchronize
|
||
// access to the list.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// The address of the entry removed from the list, or NULL if the list is
|
||
// empty.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY(ExInterlockedRemoveHeadList)
|
||
|
||
ori r.6,r.3,0 // move listhead address
|
||
|
||
DISABLE_INTERRUPTS(r.8,r.12) // disable interrupts
|
||
// r.8 <- previous msr value
|
||
// r.12 <- new (disabled) msr
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
ACQUIRE_SPIN_LOCK(r.4, r.4, r.9, remhl, remhlw)
|
||
#endif
|
||
|
||
lwz r.7,LsFlink(r.6) // get address of next entry
|
||
xor. r.3,r.7,r.6 // check if list is empty
|
||
beq nullrlist // if eq, list is empty
|
||
lwz r.10,LsFlink(r.7) // get address of next entry
|
||
ori r.3,r.7,0 // set address of entry removed
|
||
stw r.10,LsFlink(r.6) // store address of next in head
|
||
stw r.6,LsBlink(r.10) // store address of previous in next
|
||
nullrlist:
|
||
|
||
#if !defined(NT_UP)
|
||
RELEASE_SPIN_LOCK(r.4, r.9)
|
||
#endif
|
||
|
||
ENABLE_INTERRUPTS(r.8) // enable interrupts
|
||
|
||
blr // return
|
||
|
||
#if !defined(NT_UP)
|
||
SPIN_ON_SPIN_LOCK_ENABLED(r.4, r.9, remhl, remhlw, remhlx, r.8, r.12)
|
||
#endif
|
||
|
||
DUMMY_EXIT(ExInterlockedRemoveHeadList)
|
||
|
||
// SBTTL("Interlocked Pop Entry List")
|
||
//++
|
||
//
|
||
// PSINGLE_LIST_ENTRY
|
||
// ExInterlockedPopEntryList (
|
||
// IN PSINGLE_LIST_ENTRY ListHead,
|
||
// IN PKSPIN_LOCK Lock
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function removes an entry from the front of a singly linked list
|
||
// so that access to the list is synchronized in a multiprocessor system.
|
||
// If there are no entries in the list, then a value of NULL is returned.
|
||
// Otherwise, the address of the entry that is removed is returned as the
|
||
// function value.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// ListHead (r.3) - Supplies a pointer to the head of the singly linked
|
||
// list from which an entry is to be removed.
|
||
//
|
||
// Lock (r.4) - Supplies a pointer to a spin lock to be used to synchronize
|
||
// access to the list.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// The address of the entry removed from the list, or NULL if the list is
|
||
// empty.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY(ExInterlockedPopEntryList)
|
||
|
||
ori r.6,r.3,0 // move listhead address
|
||
|
||
DISABLE_INTERRUPTS(r.8,r.12) // disable interrupts
|
||
// r.8 <- previous msr value
|
||
// r.12 <- new (disabled) msr
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
ACQUIRE_SPIN_LOCK(r.4, r.4, r.9, popel, popelw)
|
||
#endif
|
||
|
||
lwz r.3,0(r.6) // get address of next entry
|
||
cmpwi r.3,0 // check if list is empty
|
||
beq nullplist // if eq, list is empty
|
||
lwz r.5,0(r.3) // get address of next entry
|
||
stw r.5,0(r.6) // store address of next in head
|
||
nullplist:
|
||
|
||
#if !defined(NT_UP)
|
||
RELEASE_SPIN_LOCK(r.4, r.9)
|
||
#endif
|
||
|
||
ENABLE_INTERRUPTS(r.8) // enable interrupts
|
||
|
||
blr // return
|
||
|
||
#if !defined(NT_UP)
|
||
SPIN_ON_SPIN_LOCK_ENABLED(r.4, r.9, popel, popelw, popelx, r.8, r.12)
|
||
#endif
|
||
|
||
DUMMY_EXIT(ExInterlockedPopEntryList)
|
||
|
||
// SBTTL("Interlocked Push Entry List")
|
||
//++
|
||
//
|
||
// PSINGLE_LIST_ENTRY
|
||
// ExInterlockedPushEntryList (
|
||
// IN PSINGLE_LIST_ENTRY ListHead,
|
||
// IN PSINGLE_LIST_ENTRY ListEntry,
|
||
// IN PKSPIN_LOCK Lock
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function inserts an entry at the head of a singly linked list
|
||
// so that access to the list is synchronized in a multiprocessor system.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// ListHead (r.3) - Supplies a pointer to the head of the singly linked
|
||
// list into which an entry is to be inserted.
|
||
//
|
||
// ListEntry (r.4) - Supplies a pointer to the entry to be inserted at the
|
||
// head of the list.
|
||
//
|
||
// Lock (r.5) - Supplies a pointer to a spin lock to be used to synchronize
|
||
// access to the list.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// Previous contents of ListHead. NULL implies list went from empty
|
||
// to not empty.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY(ExInterlockedPushEntryList)
|
||
|
||
ori r.6,r.3,0 // move listhead address
|
||
|
||
DISABLE_INTERRUPTS(r.8,r.12) // disable interrupts
|
||
// r.8 <- previous msr value
|
||
// r.12 <- new (disabled) msr
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
ACQUIRE_SPIN_LOCK(r.5, r.5, r.9, pushel, pushelw)
|
||
#endif
|
||
|
||
lwz r.3,0(r.6) // get address of first entry (return value also)
|
||
stw r.4,0(r.6) // set address of first entry
|
||
stw r.3,0(r.4) // set address of next in new entry
|
||
|
||
#if !defined(NT_UP)
|
||
RELEASE_SPIN_LOCK(r.5, r.9)
|
||
#endif
|
||
|
||
ENABLE_INTERRUPTS(r.8) // enable interrupts
|
||
|
||
blr // return
|
||
|
||
#if !defined(NT_UP)
|
||
SPIN_ON_SPIN_LOCK_ENABLED(r.5, r.9, pushel, pushelw, pushelx, r.8, r.12)
|
||
#endif
|
||
|
||
DUMMY_EXIT(ExInterlockedPushEntryList)
|
||
|
||
// SBTTL("Interlocked Pop Entry Sequenced List")
|
||
//++
|
||
//
|
||
// PSINGLE_LIST_ENTRY
|
||
// ExInterlockedPopEntrySList (
|
||
// IN PSLIST_HEADER ListHead,
|
||
// IN PKSPIN_LOCK Lock
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function removes an entry from the front of a sequenced singly
|
||
// linked list so that access to the list is synchronized in a MP system.
|
||
// If there are no entries in the list, then a value of NULL is returned.
|
||
// Otherwise, the address of the entry that is removed is returned as the
|
||
// function value.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// ListHead (r.3) - Supplies a pointer to the sequenced listhead from which
|
||
// an entry is to be removed.
|
||
//
|
||
// Lock (r.4) - Supplies a pointer to a spin lock to be used to synchronize
|
||
// access to the list.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// The address of the entry removed from the list, or NULL if the list is
|
||
// empty.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY(ExInterlockedPopEntrySList)
|
||
|
||
ori r.6,r.3,0 // move listhead address
|
||
|
||
DISABLE_INTERRUPTS(r.8,r.12) // disable interrupts
|
||
// r.8 <- previous msr value
|
||
// r.12 <- new (disabled) msr
|
||
|
||
li r.0,0 // zero r.0
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
ACQUIRE_SPIN_LOCK(r.4, r.4, r.9, popesl, popeslw)
|
||
#endif
|
||
|
||
lwz r.3,0(r.6) // get address of next entry
|
||
ori r.0,r.0,0xffff // set r.0 to 0x0000ffff
|
||
cmpwi r.3,0 // check if list is empty
|
||
lwz r.7,4(r.6) // get depth and sequence number
|
||
beq nullpslist // if eq, list is empty
|
||
|
||
lwz r.5,0(r.3) // get address of next entry
|
||
add r.7,r.7,r.0 // decrement depth and increment
|
||
// sequence number
|
||
stw r.5,0(r.6) // store address of next in head
|
||
stw r.7,4(r.6) // store new depth and sequence number
|
||
|
||
nullpslist:
|
||
|
||
#if !defined(NT_UP)
|
||
RELEASE_SPIN_LOCK(r.4, r.9)
|
||
#endif
|
||
|
||
ENABLE_INTERRUPTS(r.8) // enable interrupts
|
||
|
||
blr // return
|
||
|
||
#if !defined(NT_UP)
|
||
SPIN_ON_SPIN_LOCK_ENABLED(r.4, r.9, popesl, popeslw, popeslx, r.8, r.12)
|
||
#endif
|
||
|
||
DUMMY_EXIT(ExInterlockedPopEntrySList)
|
||
|
||
// SBTTL("Interlocked Push Entry Sequenced List")
|
||
//++
|
||
//
|
||
// PSINGLE_LIST_ENTRY
|
||
// ExInterlockedPushEntrySList (
|
||
// IN PSLIST_HEADER ListHead,
|
||
// IN PSINGLE_LIST_ENTRY ListEntry,
|
||
// IN PKSPIN_LOCK Lock
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function inserts an entry at the head of a sequenced singly linked
|
||
// list so that access to the list is synchronized in an MP system.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// ListHead (r.3) - Supplies a pointer to the sequenced listhead into which
|
||
// an entry is to be inserted.
|
||
//
|
||
// ListEntry (r.4) - Supplies a pointer to the entry to be inserted at the
|
||
// head of the list.
|
||
//
|
||
// Lock (r.5) - Supplies a pointer to a spin lock to be used to synchronize
|
||
// access to the list.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// Previous contents of ListHead. NULL implies list went from empty
|
||
// to not empty.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY(ExInterlockedPushEntrySList)
|
||
|
||
ori r.6,r.3,0 // move listhead address
|
||
|
||
DISABLE_INTERRUPTS(r.8,r.12) // disable interrupts
|
||
// r.8 <- previous msr value
|
||
// r.12 <- new (disabled) msr
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
ACQUIRE_SPIN_LOCK(r.5, r.5, r.9, pushesl, pusheslw)
|
||
#endif
|
||
|
||
lwz r.7,4(r.6) // get depth and sequence number
|
||
lwz r.3,0(r.6) // get address of first entry (return value also)
|
||
addi r.7,r.7,1 // increment depth
|
||
stw r.4,0(r.6) // set address of first entry
|
||
addis r.7,r.7,1 // increment sequence number
|
||
stw r.3,0(r.4) // set address of next in new entry
|
||
stw r.7,4(r.6) // store new depth and sequence number
|
||
|
||
#if !defined(NT_UP)
|
||
RELEASE_SPIN_LOCK(r.5, r.9)
|
||
#endif
|
||
|
||
ENABLE_INTERRUPTS(r.8) // enable interrupts
|
||
|
||
blr // return
|
||
|
||
#if !defined(NT_UP)
|
||
SPIN_ON_SPIN_LOCK_ENABLED(r.5, r.9, pushesl, pusheslw, pusheslx, r.8, r.12)
|
||
#endif
|
||
|
||
DUMMY_EXIT(ExInterlockedPushEntrySList)
|
||
|
||
// SBTTL("Interlocked Compare Exchange 64-bits")
|
||
//++
|
||
//
|
||
// ULONGLONG
|
||
// ExInterlockedCompareExchange64 (
|
||
// IN PULONGLONG Destination,
|
||
// IN PULONGLONG Exchange,
|
||
// IN PULONGLONG Comparand,
|
||
// IN PKSPIN_LOCK Lock
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function performs an interlocked compare and exchange of 64-bits.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// Destination (r3) - Supplies a pointer to the destination variable.
|
||
//
|
||
// Exchange (r4) - Supplies a pointer to the exchange value.
|
||
//
|
||
// Comparand (r5) - Supplies a pointer to the comparand value.
|
||
//
|
||
// Lock (r6) - Supplies a pointer to a spin lock to use to synchronize
|
||
// access to Destination.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// The current destination value is returned as the function value
|
||
// (in r3 and r4).
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY(ExInterlockedCompareExchange64)
|
||
|
||
lwz r.0,0(r.5) // get comparand (low)
|
||
lwz r.5,4(r.5) // get comparand (high)
|
||
|
||
DISABLE_INTERRUPTS(r.8,r.12) // disable interrupts
|
||
// r.8 <- previous msr value
|
||
// r.12 <- new (disabled) msr
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
ACQUIRE_SPIN_LOCK(r.6, r.6, r.9, cmpex64, cmpex64w)
|
||
#endif
|
||
|
||
lwz r.10,0(r.3) // get current value (low)
|
||
lwz r.11,4(r.3) // get current value (high)
|
||
cmpw cr.0,r.0,r.10 // compare current with comparand (low)
|
||
cmpw cr.1,r.5,r.11 // compare current with comparand (high)
|
||
bne cr.0,cmpex64_no // if ne, current and comparand mismatch (low)
|
||
bne cr.1,cmpex64_no // if ne, current and comparand mismatch (high)
|
||
|
||
lwz r.0,0(r.4) // get exchange value (low)
|
||
lwz r.5,4(r.4) // get exchange value (high)
|
||
stw r.0,0(r.3) // store exchange value (low)
|
||
stw r.5,4(r.3) // store exchange value (high)
|
||
|
||
cmpex64_no:
|
||
|
||
#if !defined(NT_UP)
|
||
RELEASE_SPIN_LOCK(r.6, r.9)
|
||
#endif
|
||
|
||
ENABLE_INTERRUPTS(r.8) // enable interrupts
|
||
|
||
ori r.3,r.10,0 // return current value (low)
|
||
ori r.4,r.11,0 // return current value (low)
|
||
|
||
blr // return
|
||
|
||
#if !defined(NT_UP)
|
||
SPIN_ON_SPIN_LOCK_ENABLED(r.6, r.9, cmpex64, cmpex64w, cmpex64x, r.8, r.12)
|
||
#endif
|
||
|
||
DUMMY_EXIT(ExInterlockedCompareExchange64)
|
||
|
||
#if COLLECT_PAGING_DATA
|
||
|
||
.struct 0
|
||
.space StackFrameHeaderLength
|
||
gpiLr: .space 4
|
||
gpiR31: .space 4
|
||
gpiR30: .space 4
|
||
gpiR29: .space 4
|
||
gpiR28: .space 4
|
||
gpiR27: .space 4
|
||
gpiR26: .space 4
|
||
gpiR25: .space 4
|
||
gpiR24: .space 4
|
||
.align 3
|
||
gpiFrameLength:
|
||
|
||
SPECIAL_ENTRY(ExpGetPagingInformation)
|
||
|
||
mflr r0
|
||
stwu sp, -gpiFrameLength(sp)
|
||
stw r31, gpiR31(sp)
|
||
stw r30, gpiR30(sp)
|
||
stw r29, gpiR29(sp)
|
||
stw r28, gpiR28(sp)
|
||
stw r27, gpiR27(sp)
|
||
stw r26, gpiR26(sp)
|
||
stw r25, gpiR25(sp)
|
||
stw r24, gpiR24(sp)
|
||
stw r0, gpiLr(sp)
|
||
|
||
PROLOGUE_END(ExpGetPagingInformation)
|
||
|
||
ori r30, r3, 0
|
||
ori r31, r4, 0
|
||
|
||
lwz r29, [toc]KeNumberProcessors(r2)
|
||
lwz r29, 0(r29)
|
||
stw r29, 0(r30)
|
||
|
||
mfsdr1 r28
|
||
addi r27, r28, 1
|
||
rlwinm r27, r27, 16, 0x03ff0000
|
||
stw r27, 4(r30)
|
||
rlwinm r24, r27, 31, 0x7fffffff
|
||
|
||
subi r31, r31, 8
|
||
addi r30, r30, 8
|
||
|
||
lwz r26, [toc]KiProcessorBlock(r2)
|
||
|
||
gpi_procloop:
|
||
|
||
lwz r25, 0(r26)
|
||
lwz r25, PbPcrPage(r25)
|
||
slwi r25, r25, PAGE_SHIFT
|
||
oris r25, r25, 0x8000
|
||
|
||
addi r26, r26, 4
|
||
subi r29, r29, 1
|
||
|
||
subic. r31, r31, CTR_SIZE
|
||
blt skip_processor_data
|
||
ori r3, r30, 0
|
||
la r4, PcPagingData(r25)
|
||
li r5, CTR_SIZE
|
||
bl ..RtlCopyMemory
|
||
|
||
skip_processor_data:
|
||
|
||
addi r30, r30, CTR_SIZE
|
||
|
||
cmpwi r29, 0
|
||
bne gpi_procloop
|
||
|
||
sub. r31, r31, r27
|
||
blt skip_hpt
|
||
ori r3, r30, 0
|
||
rlwinm r28, r28, 0, 0xffff0000
|
||
oris r4, r28, 0x8000
|
||
ori r5, r27, 0
|
||
bl ..RtlCopyMemory
|
||
|
||
skip_hpt:
|
||
|
||
ori r3, r31, 0
|
||
|
||
lwz r0, gpiLr(sp)
|
||
lwz r31, gpiR31(sp)
|
||
lwz r30, gpiR30(sp)
|
||
lwz r29, gpiR29(sp)
|
||
lwz r28, gpiR28(sp)
|
||
lwz r27, gpiR27(sp)
|
||
lwz r26, gpiR26(sp)
|
||
lwz r25, gpiR25(sp)
|
||
lwz r24, gpiR24(sp)
|
||
mtlr r0
|
||
addi sp, sp, gpiFrameLength
|
||
|
||
SPECIAL_EXIT(ExpGetPagingInformation)
|
||
|
||
#endif // COLLECT_PAGING_DATA
|