xbox-kernel/private/ntos/ex/i386/intrlfst.asm
2020-09-30 17:17:25 +02:00

816 lines
22 KiB
NASM

title "Interlocked Support"
;++
;
; Copyright (c) 1989 Microsoft Corporation
;
; Module Name:
;
; intrlfst.asm
;
; Abstract:
;
; This module implements functions to support interlocked operations.
; Interlocked operations can only operate on nonpaged data.
;
; This module implements the fast call version of the interlocked
; fuctions.
;
; Author:
;
; Ken Reneris (kenr) 5-May-1994
;
; Environment:
;
; Any mode.
;
; Revision History:
;
;--
.386p
.xlist
include ks386.inc
include callconv.inc ; calling convention macros
.list
_TEXT SEGMENT DWORD PUBLIC 'CODE'
ASSUME DS:FLAT, ES:FLAT, SS:NOTHING, FS:NOTHING, GS:NOTHING
;++
;
; General Notes on Interlocked Procedures:
;
; These procedures assume that neither their code, nor any of
; the data they touch, will cause a page fault.
;
; They use spinlocks to achieve MP atomicity, iff it's an MP machine.
; (The spinlock macros generate zilch if NT_UP = 1, and
; we if out some aux code here as well.)
;
; They turn off interrupts so that they can be used for synchronization
; between ISRs and driver code. Flags are preserved so they can
; be called in special code (Like IPC interrupt handlers) that
; may have interrupts off.
;
;--
page ,132
subttl "ExInterlockedAddLargeStatistic"
;++
;
; VOID
; FASTCALL
; ExInterlockedAddLargeStatistic (
; IN PLARGE_INTEGER Addend,
; IN ULONG Increment
; )
;
; Routine Description:
;
; This function performs an interlocked add of an increment value to an
; addend variable of type unsigned large integer.
;
; Arguments:
;
; (ecx) Addend - Supplies a pointer to the variable whose value is
; adjusted by the increment value.
;
; (edx) Increment - Supplies the increment value that is added to the
; addend variable.
;
; Return Value:
;
; None.
;
;--
cPublicFastCall ExInterlockedAddLargeStatistic, 2
cPublicFpo 0,0
add dword ptr [ecx], edx ; add low part of large statistic
adc dword ptr [ecx+4], 0 ; add carry to high part
fstRET ExInterlockedAddLargeStatistic ; return
fstENDP ExInterlockedAddLargeStatistic
page , 132
subttl "Interlocked Insert Head List"
;++
;
; PLIST_ENTRY
; ExfInterlockedInsertHeadList (
; IN PLIST_ENTRY ListHead,
; IN PLIST_ENTRY ListEntry
; )
;
; Routine Description:
;
; This function inserts an entry at the head of a doubly linked list
; so that access to the list is synchronized in a multiprocessor system.
;
; N.B. The pages of data which this routine operates on MUST be
; present. No page fault is allowed in this routine.
;
; Arguments:
;
; (ecx) = ListHead - Supplies a pointer to the head of the doubly linked
; list into which an entry is to be inserted.
;
; (edx) = ListEntry - Supplies a pointer to the entry to be inserted at the
; head of the list.
;
; Return Value:
;
; Pointer to entry that was at the head of the list or NULL if the list
; was empty.
;
;--
cPublicFastCall ExfInterlockedInsertHeadList , 2
cPublicFpo 0, 1
pushfd
cli
mov eax, LsFlink[ecx] ; (eax)->next entry in the list
mov [edx]+LsFlink, eax ; store next link in entry
mov [edx]+LsBlink, ecx ; store previous link in entry
mov [ecx]+LsFlink, edx ; store next link in head
mov [eax]+LsBlink, edx ; store previous link in next
popfd
xor eax, ecx ; return null if list was empty
jz short Eiih15
xor eax, ecx
Eiih15: fstRET ExfInterlockedInsertHeadList
fstENDP ExfInterlockedInsertHeadList
page , 132
subttl "Interlocked Insert Tail List"
;++
;
; PLIST_ENTRY
; FASTCALL
; ExfInterlockedInsertTailList (
; IN PLIST_ENTRY ListHead,
; IN PLIST_ENTRY ListEntry
; )
;
; Routine Description:
;
; This function inserts an entry at the tail of a doubly linked list
; so that access to the list is synchronized in a multiprocessor system.
;
; N.B. The pages of data which this routine operates on MUST be
; present. No page fault is allowed in this routine.
;
; Arguments:
;
; (ecx) = ListHead - Supplies a pointer to the head of the doubly linked
; list into which an entry is to be inserted.
;
; (edx) = ListEntry - Supplies a pointer to the entry to be inserted at the
; tail of the list.
;
; Return Value:
;
; Pointer to entry that was at the tail of the list or NULL if the list
; was empty.
;
;--
cPublicFastCall ExfInterlockedInsertTailList, 2
cPublicFpo 0, 1
pushfd
cli
mov eax, LsBlink[ecx] ; (eax)->prev entry in the list
mov [edx]+LsFlink, ecx ; store next link in entry
mov [edx]+LsBlink, eax ; store previous link in entry
mov [ecx]+LsBlink, edx ; store next link in head
mov [eax]+LsFlink, edx ; store previous link in next
popfd
xor eax, ecx ; return null if list was empty
jz short Eiit15
xor eax, ecx
Eiit15: fstRET ExfInterlockedInsertTailList
fstENDP ExfInterlockedInsertTailList
page , 132
subttl "Interlocked Remove Head List"
;++
;
; PLIST_ENTRY
; FASTCALL
; ExfInterlockedRemoveHeadList (
; IN PLIST_ENTRY ListHead
; )
;
; Routine Description:
;
; This function removes an entry from the head of a doubly linked list
; so that access to the list is synchronized in a multiprocessor system.
; If there are no entries in the list, then a value of NULL is returned.
; Otherwise, the address of the entry that is removed is returned as the
; function value.
;
; N.B. The pages of data which this routine operates on MUST be
; present. No page fault is allowed in this routine.
;
; Arguments:
;
; (ecx) ListHead - Supplies a pointer to the head of the doubly linked
; list from which an entry is to be removed.
;
; Return Value:
;
; The address of the entry removed from the list, or NULL if the list is
; empty.
;
;--
cPublicFastCall ExfInterlockedRemoveHeadList , 1
cPublicFpo 0, 1
pushfd
cli
mov eax, [ecx]+LsFlink ; (eax)-> next entry
cmp eax, ecx ; Is list empty?
je short Eirh20 ; if e, list is empty, go Eirh20
mov edx, [eax]+LsFlink ; (ecx)-> next entry(after deletion)
mov [ecx]+LsFlink, edx ; store address of next in head
mov [edx]+LsBlink, ecx ; store address of previous in next
if DBG
mov [eax]+LsFlink, 0baddd0ffh
mov [eax]+LsBlink, 0baddd0ffh
endif
popfd ; restore flags including interrupts
fstRET ExfInterlockedRemoveHeadList
Eirh20: popfd
xor eax,eax ; (eax) = null for empty list
fstRET ExfInterlockedRemoveHeadList
fstENDP ExfInterlockedRemoveHeadList
page , 132
subttl "Interlocked Pop Entry Sequenced List"
;++
;
; PSINGLE_LIST_ENTRY
; FASTCALL
; InterlockedPopEntrySList (
; IN PSINGLE_LIST_ENTRY ListHead
; )
;
; Routine Description:
;
; This function removes an entry from the front of a sequenced singly
; linked list so that access to the list is synchronized in an MP system.
; If there are no entries in the list, then a value of NULL is returned.
; Otherwise, the address of the entry that is removed is returned as the
; function value.
;
; N.B. The cmpxchg8b instruction is only supported on some processors.
; If the host processor does not support this instruction, then
; then following code is patched to contain a jump to the normal
; pop entry code which has a compatible calling sequence and data
; structure.
;
; Arguments:
;
; (ecx) = ListHead - Supplies a pointer to the sequenced listhead from
; which an entry is to be removed.
;
; Return Value:
;
; The address of the entry removed from the list, or NULL if the list is
; empty.
;
;--
cPublicFastCall InterlockedPopEntrySList, 1
cPublicFpo 0,2
;
; Save nonvolatile registers and read the listhead sequence number followed
; by the listhead next link.
;
; N.B. These two dwords MUST be read exactly in this order.
;
push ebx ; save nonvolatile registers
push ebp ;
mov ebp, ecx ; save listhead address
;
; N.B. The following code is the continuation address should a fault
; occur in the rare case described below.
;
public ExpInterlockedPopEntrySListResume
ExpInterlockedPopEntrySListResume: ;
mov edx, [ebp] + 4 ; get current sequence number
mov eax, [ebp] + 0 ; get current next link
;
; If the list is empty, then there is nothing that can be removed.
;
Epop10: or eax, eax ; check if list is empty
jz short Epop20 ; if z set, list is empty
mov ecx, edx ; copy sequence number and depth
add ecx, 0FFFFH ; adjust sequence number and depth
;
; N.B. It is possible for the following instruction to fault in the rare
; case where the first entry in the list is allocated on another
; processor and freed between the time the free pointer is read above
; and the following instruction. When this happens, the access fault
; code continues execution above at the resumption address and the
; entire operation is retried.
;
public ExpInterlockedPopEntrySListFault
ExpInterlockedPopEntrySListFault: ;
mov ebx, [eax] ; get address of successor entry
.586
cmpxchg8b qword ptr [ebp] ; compare and exchange
.386
jnz short Epop10 ; if z clear, exchange failed
;
; Restore nonvolatile registers and return result.
;
cPublicFpo 0,0
Epop20: pop ebp ; restore nonvolatile registers
pop ebx ;
fstRET InterlockedPopEntrySList
fstENDP InterlockedPopEntrySList
page , 132
subttl "Interlocked Push Entry Sequenced List"
;++
;
; PSINGLE_LIST_ENTRY
; InterlockedPushEntrySList (
; IN PSINGLE_LIST_ENTRY ListHead,
; IN PSINGLE_LIST_ENTRY ListEntry
; )
;
; Routine Description:
;
; This function inserts an entry at the head of a sequenced singly linked
; list so that access to the list is synchronized in an MP system.
;
; N.B. The cmpxchg8b instruction is only supported on some processors.
; If the host processor does not support this instruction, then
; then following code is patched to contain a jump to the normal
; push entry code which has a compatible calling sequence and data
; structure.
;
; Arguments:
;
; (ecx) ListHead - Supplies a pointer to the sequenced listhead into which
; an entry is to be inserted.
;
; (edx) ListEntry - Supplies a pointer to the entry to be inserted at the
; head of the list.
;
; Return Value:
;
; Previous contents of ListHead. NULL implies list went from empty
; to not empty.
;
;--
cPublicFastCall InterlockedPushEntrySList, 2
cPublicFpo 0,2
;
; Save nonvolatile registers and read the listhead sequence number followed
; by the listhead next link.
;
; N.B. These two dwords MUST be read exactly in this order.
;
push ebx ; save nonvolatile registers
push ebp ;
mov ebp, ecx ; save listhead address
mov ebx, edx ; save list entry address
mov edx, [ebp] + 4 ; get current sequence number
mov eax, [ebp] + 0 ; get current next link
Epsh10: mov [ebx], eax ; set next link in new first entry
mov ecx, edx ; copy sequence number
add ecx, 010001H ; increment sequence number and depth
.586
cmpxchg8b qword ptr[ebp] ; compare and exchange
.386
jnz short Epsh10 ; if z clear, exchange failed
;
; Restore nonvolatile registers and return result.
;
cPublicFpo 0,0
pop ebp ; restore nonvolatile registers
pop ebx ;
fstRET InterlockedPushEntrySList
fstENDP InterlockedPushEntrySList
page , 132
subttl "Interlocked Flush Sequenced List"
;++
;
; PSINGLE_LIST_ENTRY
; FASTCALL
; InterlockedFlushSList (
; IN PSINGLE_LIST_ENTRY ListHead
; )
;
; Routine Description:
;
; This function removes the entire list from a sequenced singly
; linked list so that access to the list is synchronized in an MP system.
; If there are no entries in the list, then a value of NULL is returned.
; Otherwise, the address of the entry at the top of the list is removed
; and returned as the function value and the list header is set to point
; to NULL.
;
; N.B. The cmpxchg8b instruction is only supported on some processors.
; If the host processor does not support this instruction, then
; then following code is patched to contain a jump to the normal
; pop entry code which has a compatible calling sequence and data
; structure.
;
; Arguments:
;
; (ecx) = ListHead - Supplies a pointer to the sequenced listhead from
; which the list is to be flushed.
;
; Return Value:
;
; The address of the entire current list, or NULL if the list is
; empty.
;
;--
cPublicFastCall InterlockedFlushSList, 1
cPublicFpo 0,1
;
; Save nonvolatile registers and read the listhead sequence number followed
; by the listhead next link.
;
; N.B. These two dwords MUST be read exactly in this order.
;
push ebx ; save nonvolatile registers
push ebp ;
mov ebp, ecx ; save listhead address
mov edx, [ebp] + 4 ; get current sequence number
mov eax, [ebp] + 0 ; get current next link
;
; N.B. The following code is the retry code should the compare
; part of the compare exchange operation fail
;
; If the list is empty, then there is nothing that can be removed.
;
Efls10: or eax, eax ; check if list is empty
jz short Efls20 ; if z set, list is empty
mov ecx, 0 ; clear sequence number and depth
mov ebx, 0 ; clear successor entry pointer
.586
cmpxchg8b qword ptr [ebp] ; compare and exchange
.386
jnz short Efls10 ; if z clear, exchange failed
;
; Restore nonvolatile registers and return result.
;
cPublicFpo 0,0
Efls20: pop ebp ; restore nonvolatile registers
pop ebx ;
fstRET InterlockedFlushSList
fstENDP InterlockedFlushSList
page , 132
subttl "Interlocked i386 Exchange Ulong"
;++
;
; ULONG
; FASTCALL
; InterlockedExchange (
; IN PULONG Target,
; IN ULONG Value
; )
;
; Routine Description:
;
; This function atomically exchanges the Target and Value, returning
; the prior contents of Target
;
; See Exi386InterlockedExchangeUlong. This function is the i386
; architectural specific version of ExInterlockedDecrementLong.
; No source directly calls this function, instead
; ExInterlockedDecrementLong is called and when built on x86 these
; calls are macroed to the i386 optimized version.
;
; Arguments:
;
; (ecx) = Source - Address of ULONG to exchange
; (edx) = Value - New value of ULONG
;
; Return Value:
;
; The prior value of Source
;--
cPublicFastCall InterlockedExchange, 2
cPublicFpo 0,0
.486
mov eax, [ecx] ; get comperand value
Ixchg: cmpxchg [ecx], edx ; compare and swap
jnz Ixchg ; if nz, exchange failed
.386
fstRET InterlockedExchange
fstENDP InterlockedExchange
;++
;
; LONG
; InterlockedIncrement(
; IN PLONG Addend
; )
;
; Routine Description:
;
; This function performs an interlocked add of one to the addend variable.
;
; No checking is done for overflow.
;
; Arguments:
;
; Addend (ecx) - Supplies a pointer to a variable whose value is to be
; incremented by one.
;
; Return Value:
;
; (eax) - The incremented value.
;
;--
cPublicFastCall InterlockedIncrement,1
cPublicFpo 0,0
mov eax, 1 ; set increment value
.486
xadd [ecx], eax ; interlocked increment
.386p
inc eax ; adjust return value
fstRET InterlockedIncrement
fstENDP InterlockedIncrement
page , 132
subttl "InterlockedDecrment"
;++
;
; LONG
; InterlockedDecrement(
; IN PLONG Addend
; )
;
; Routine Description:
;
; This function performs an interlocked add of -1 to the addend variable.
;
; No checking is done for overflow
;
; Arguments:
;
; Addend (ecx) - Supplies a pointer to a variable whose value is to be
; decremented by one.
;
; Return Value:
;
; (eax) - The decremented value.
;
;--
cPublicFastCall InterlockedDecrement,1
cPublicFpo 0,0
mov eax, -1 ; set decrment value
.486
xadd [ecx], eax ; interlocked decrement
.386
dec eax ; adjust return value
fstRET InterlockedDecrement
fstENDP InterlockedDecrement
page , 132
subttl "Interlocked Compare Exchange"
;++
;
; PVOID
; FASTCALL
; InterlockedCompareExchange (
; IN OUT PVOID *Destination,
; IN PVOID Exchange,
; IN PVOID Comperand
; )
;
; Routine Description:
;
; This function performs an interlocked compare of the destination
; value with the comperand value. If the destination value is equal
; to the comperand value, then the exchange value is stored in the
; destination. Otherwise, no operation is performed.
;
; Arguments:
;
; (ecx) Destination - Supplies a pointer to destination value.
;
; (edx) Exchange - Supplies the exchange value.
;
; [esp + 4] Comperand - Supplies the comperand value.
;
; Return Value:
;
; The initial destination value is returned as the function value.
;
;--
cPublicFastCall InterlockedCompareExchange, 3
cPublicFpo 0,0
mov eax, [esp + 4] ; set comperand value
.486
cmpxchg [ecx], edx ; compare and exchange
.386
fstRET InterlockedCompareExchange
fstENDP InterlockedCompareExchange
page , 132
subttl "Interlocked Compare Exchange 64-bits"
;++
;
; LONGLONG
; FASTCALL
; ExInterlockedCompareExchange64 (
; IN PLONGLONG Destination,
; IN PLONGLONG Exchange,
; IN PLONGLONG Comperand
; )
;
; Routine Description:
;
; This function performs a compare and exchange of 64-bits.
;
; N.B. The cmpxchg8b instruction is only supported on some processors.
; If the host processor does not support this instruction, then
; then following code is patched to contain a jump to the normal
; compare exchange 64-bit code which has a compatible calling
; sequence and data structure.
;
; Arguments:
;
; (ecx) Destination - Supplies a pointer to the destination variable.
;
; (edx) Exchange - Supplies a pointer to the exchange value.
;
; (esp+4) Comperand - Supplies a pointer to the comperand value.
;
; Return Value:
;
; The current destination value is returned as the function value.
;
;--
cPublicFastCall ExInterlockedCompareExchange64, 3
cPublicFpo 0,1
;
; Save nonvolatile registers and read the exchange and comperand values.
;
push ebx ; save nonvolatile registers
push ebp ;
mov ebp, ecx ; set destination address
mov ebx, [edx] ; get exchange value
mov ecx, [edx] + 4 ;
mov edx, [esp] + 12 ; get comperand address
mov eax, [edx] ; get comperand value
mov edx, [edx] + 4 ;
.586
cmpxchg8b qword ptr[ebp] ; compare and exchange
.386
;
; Restore nonvolatile registers and return result in edx:eax.
;
cPublicFpo 0,0
pop ebp ; restore nonvolatile registers
pop ebx ;
fstRET ExInterlockedCompareExchange64
fstENDP ExInterlockedCompareExchange64
page , 132
subttl "Interlocked Exchange Add"
;++
;
; LONG
; FASTCALL
; InterlockedExchangeAdd (
; IN OUT PLONG Addend,
; IN LONG Increment
; )
;
; Routine Description:
;
; This function performs an interlocked add of an increment value to an
; addend variable of type unsinged long. The initial value of the addend
; variable is returned as the function value.
;
; It is NOT possible to mix ExInterlockedDecrementLong and
; ExInterlockedIncrementong with ExInterlockedAddUlong.
;
;
; Arguments:
;
; (ecx) Addend - Supplies a pointer to a variable whose value is to be
; adjusted by the increment value.
;
; (edx) Increment - Supplies the increment value to be added to the
; addend variable.
;
; Return Value:
;
; The initial value of the addend variable.
;
;--
cPublicFastCall InterlockedExchangeAdd, 2
cPublicFpo 0,0
.486
xadd [ecx], edx ; exchange add
.386
mov eax, edx ; set initial value
fstRET InterlockedExchangeAdd
fstENDP InterlockedExchangeAdd
_TEXT ends
end