title "Interlocked Support" ;++ ; ; Copyright (c) 1989 Microsoft Corporation ; ; Module Name: ; ; intrlfst.asm ; ; Abstract: ; ; This module implements functions to support interlocked operations. ; Interlocked operations can only operate on nonpaged data. ; ; This module implements the fast call version of the interlocked ; fuctions. ; ; Author: ; ; Ken Reneris (kenr) 5-May-1994 ; ; Environment: ; ; Any mode. ; ; Revision History: ; ;-- .386p .xlist include ks386.inc include callconv.inc ; calling convention macros .list _TEXT SEGMENT DWORD PUBLIC 'CODE' ASSUME DS:FLAT, ES:FLAT, SS:NOTHING, FS:NOTHING, GS:NOTHING ;++ ; ; General Notes on Interlocked Procedures: ; ; These procedures assume that neither their code, nor any of ; the data they touch, will cause a page fault. ; ; They use spinlocks to achieve MP atomicity, iff it's an MP machine. ; (The spinlock macros generate zilch if NT_UP = 1, and ; we if out some aux code here as well.) ; ; They turn off interrupts so that they can be used for synchronization ; between ISRs and driver code. Flags are preserved so they can ; be called in special code (Like IPC interrupt handlers) that ; may have interrupts off. ; ;-- page ,132 subttl "ExInterlockedAddLargeStatistic" ;++ ; ; VOID ; FASTCALL ; ExInterlockedAddLargeStatistic ( ; IN PLARGE_INTEGER Addend, ; IN ULONG Increment ; ) ; ; Routine Description: ; ; This function performs an interlocked add of an increment value to an ; addend variable of type unsigned large integer. ; ; Arguments: ; ; (ecx) Addend - Supplies a pointer to the variable whose value is ; adjusted by the increment value. ; ; (edx) Increment - Supplies the increment value that is added to the ; addend variable. ; ; Return Value: ; ; None. ; ;-- cPublicFastCall ExInterlockedAddLargeStatistic, 2 cPublicFpo 0,0 add dword ptr [ecx], edx ; add low part of large statistic adc dword ptr [ecx+4], 0 ; add carry to high part fstRET ExInterlockedAddLargeStatistic ; return fstENDP ExInterlockedAddLargeStatistic page , 132 subttl "Interlocked Insert Head List" ;++ ; ; PLIST_ENTRY ; ExfInterlockedInsertHeadList ( ; IN PLIST_ENTRY ListHead, ; IN PLIST_ENTRY ListEntry ; ) ; ; Routine Description: ; ; This function inserts an entry at the head of a doubly linked list ; so that access to the list is synchronized in a multiprocessor system. ; ; N.B. The pages of data which this routine operates on MUST be ; present. No page fault is allowed in this routine. ; ; Arguments: ; ; (ecx) = ListHead - Supplies a pointer to the head of the doubly linked ; list into which an entry is to be inserted. ; ; (edx) = ListEntry - Supplies a pointer to the entry to be inserted at the ; head of the list. ; ; Return Value: ; ; Pointer to entry that was at the head of the list or NULL if the list ; was empty. ; ;-- cPublicFastCall ExfInterlockedInsertHeadList , 2 cPublicFpo 0, 1 pushfd cli mov eax, LsFlink[ecx] ; (eax)->next entry in the list mov [edx]+LsFlink, eax ; store next link in entry mov [edx]+LsBlink, ecx ; store previous link in entry mov [ecx]+LsFlink, edx ; store next link in head mov [eax]+LsBlink, edx ; store previous link in next popfd xor eax, ecx ; return null if list was empty jz short Eiih15 xor eax, ecx Eiih15: fstRET ExfInterlockedInsertHeadList fstENDP ExfInterlockedInsertHeadList page , 132 subttl "Interlocked Insert Tail List" ;++ ; ; PLIST_ENTRY ; FASTCALL ; ExfInterlockedInsertTailList ( ; IN PLIST_ENTRY ListHead, ; IN PLIST_ENTRY ListEntry ; ) ; ; Routine Description: ; ; This function inserts an entry at the tail of a doubly linked list ; so that access to the list is synchronized in a multiprocessor system. ; ; N.B. The pages of data which this routine operates on MUST be ; present. No page fault is allowed in this routine. ; ; Arguments: ; ; (ecx) = ListHead - Supplies a pointer to the head of the doubly linked ; list into which an entry is to be inserted. ; ; (edx) = ListEntry - Supplies a pointer to the entry to be inserted at the ; tail of the list. ; ; Return Value: ; ; Pointer to entry that was at the tail of the list or NULL if the list ; was empty. ; ;-- cPublicFastCall ExfInterlockedInsertTailList, 2 cPublicFpo 0, 1 pushfd cli mov eax, LsBlink[ecx] ; (eax)->prev entry in the list mov [edx]+LsFlink, ecx ; store next link in entry mov [edx]+LsBlink, eax ; store previous link in entry mov [ecx]+LsBlink, edx ; store next link in head mov [eax]+LsFlink, edx ; store previous link in next popfd xor eax, ecx ; return null if list was empty jz short Eiit15 xor eax, ecx Eiit15: fstRET ExfInterlockedInsertTailList fstENDP ExfInterlockedInsertTailList page , 132 subttl "Interlocked Remove Head List" ;++ ; ; PLIST_ENTRY ; FASTCALL ; ExfInterlockedRemoveHeadList ( ; IN PLIST_ENTRY ListHead ; ) ; ; Routine Description: ; ; This function removes an entry from the head of a doubly linked list ; so that access to the list is synchronized in a multiprocessor system. ; If there are no entries in the list, then a value of NULL is returned. ; Otherwise, the address of the entry that is removed is returned as the ; function value. ; ; N.B. The pages of data which this routine operates on MUST be ; present. No page fault is allowed in this routine. ; ; Arguments: ; ; (ecx) ListHead - Supplies a pointer to the head of the doubly linked ; list from which an entry is to be removed. ; ; Return Value: ; ; The address of the entry removed from the list, or NULL if the list is ; empty. ; ;-- cPublicFastCall ExfInterlockedRemoveHeadList , 1 cPublicFpo 0, 1 pushfd cli mov eax, [ecx]+LsFlink ; (eax)-> next entry cmp eax, ecx ; Is list empty? je short Eirh20 ; if e, list is empty, go Eirh20 mov edx, [eax]+LsFlink ; (ecx)-> next entry(after deletion) mov [ecx]+LsFlink, edx ; store address of next in head mov [edx]+LsBlink, ecx ; store address of previous in next if DBG mov [eax]+LsFlink, 0baddd0ffh mov [eax]+LsBlink, 0baddd0ffh endif popfd ; restore flags including interrupts fstRET ExfInterlockedRemoveHeadList Eirh20: popfd xor eax,eax ; (eax) = null for empty list fstRET ExfInterlockedRemoveHeadList fstENDP ExfInterlockedRemoveHeadList page , 132 subttl "Interlocked Pop Entry Sequenced List" ;++ ; ; PSINGLE_LIST_ENTRY ; FASTCALL ; InterlockedPopEntrySList ( ; IN PSINGLE_LIST_ENTRY ListHead ; ) ; ; Routine Description: ; ; This function removes an entry from the front of a sequenced singly ; linked list so that access to the list is synchronized in an MP system. ; If there are no entries in the list, then a value of NULL is returned. ; Otherwise, the address of the entry that is removed is returned as the ; function value. ; ; N.B. The cmpxchg8b instruction is only supported on some processors. ; If the host processor does not support this instruction, then ; then following code is patched to contain a jump to the normal ; pop entry code which has a compatible calling sequence and data ; structure. ; ; Arguments: ; ; (ecx) = ListHead - Supplies a pointer to the sequenced listhead from ; which an entry is to be removed. ; ; Return Value: ; ; The address of the entry removed from the list, or NULL if the list is ; empty. ; ;-- cPublicFastCall InterlockedPopEntrySList, 1 cPublicFpo 0,2 ; ; Save nonvolatile registers and read the listhead sequence number followed ; by the listhead next link. ; ; N.B. These two dwords MUST be read exactly in this order. ; push ebx ; save nonvolatile registers push ebp ; mov ebp, ecx ; save listhead address ; ; N.B. The following code is the continuation address should a fault ; occur in the rare case described below. ; public ExpInterlockedPopEntrySListResume ExpInterlockedPopEntrySListResume: ; mov edx, [ebp] + 4 ; get current sequence number mov eax, [ebp] + 0 ; get current next link ; ; If the list is empty, then there is nothing that can be removed. ; Epop10: or eax, eax ; check if list is empty jz short Epop20 ; if z set, list is empty mov ecx, edx ; copy sequence number and depth add ecx, 0FFFFH ; adjust sequence number and depth ; ; N.B. It is possible for the following instruction to fault in the rare ; case where the first entry in the list is allocated on another ; processor and freed between the time the free pointer is read above ; and the following instruction. When this happens, the access fault ; code continues execution above at the resumption address and the ; entire operation is retried. ; public ExpInterlockedPopEntrySListFault ExpInterlockedPopEntrySListFault: ; mov ebx, [eax] ; get address of successor entry .586 cmpxchg8b qword ptr [ebp] ; compare and exchange .386 jnz short Epop10 ; if z clear, exchange failed ; ; Restore nonvolatile registers and return result. ; cPublicFpo 0,0 Epop20: pop ebp ; restore nonvolatile registers pop ebx ; fstRET InterlockedPopEntrySList fstENDP InterlockedPopEntrySList page , 132 subttl "Interlocked Push Entry Sequenced List" ;++ ; ; PSINGLE_LIST_ENTRY ; InterlockedPushEntrySList ( ; IN PSINGLE_LIST_ENTRY ListHead, ; IN PSINGLE_LIST_ENTRY ListEntry ; ) ; ; Routine Description: ; ; This function inserts an entry at the head of a sequenced singly linked ; list so that access to the list is synchronized in an MP system. ; ; N.B. The cmpxchg8b instruction is only supported on some processors. ; If the host processor does not support this instruction, then ; then following code is patched to contain a jump to the normal ; push entry code which has a compatible calling sequence and data ; structure. ; ; Arguments: ; ; (ecx) ListHead - Supplies a pointer to the sequenced listhead into which ; an entry is to be inserted. ; ; (edx) ListEntry - Supplies a pointer to the entry to be inserted at the ; head of the list. ; ; Return Value: ; ; Previous contents of ListHead. NULL implies list went from empty ; to not empty. ; ;-- cPublicFastCall InterlockedPushEntrySList, 2 cPublicFpo 0,2 ; ; Save nonvolatile registers and read the listhead sequence number followed ; by the listhead next link. ; ; N.B. These two dwords MUST be read exactly in this order. ; push ebx ; save nonvolatile registers push ebp ; mov ebp, ecx ; save listhead address mov ebx, edx ; save list entry address mov edx, [ebp] + 4 ; get current sequence number mov eax, [ebp] + 0 ; get current next link Epsh10: mov [ebx], eax ; set next link in new first entry mov ecx, edx ; copy sequence number add ecx, 010001H ; increment sequence number and depth .586 cmpxchg8b qword ptr[ebp] ; compare and exchange .386 jnz short Epsh10 ; if z clear, exchange failed ; ; Restore nonvolatile registers and return result. ; cPublicFpo 0,0 pop ebp ; restore nonvolatile registers pop ebx ; fstRET InterlockedPushEntrySList fstENDP InterlockedPushEntrySList page , 132 subttl "Interlocked Flush Sequenced List" ;++ ; ; PSINGLE_LIST_ENTRY ; FASTCALL ; InterlockedFlushSList ( ; IN PSINGLE_LIST_ENTRY ListHead ; ) ; ; Routine Description: ; ; This function removes the entire list from a sequenced singly ; linked list so that access to the list is synchronized in an MP system. ; If there are no entries in the list, then a value of NULL is returned. ; Otherwise, the address of the entry at the top of the list is removed ; and returned as the function value and the list header is set to point ; to NULL. ; ; N.B. The cmpxchg8b instruction is only supported on some processors. ; If the host processor does not support this instruction, then ; then following code is patched to contain a jump to the normal ; pop entry code which has a compatible calling sequence and data ; structure. ; ; Arguments: ; ; (ecx) = ListHead - Supplies a pointer to the sequenced listhead from ; which the list is to be flushed. ; ; Return Value: ; ; The address of the entire current list, or NULL if the list is ; empty. ; ;-- cPublicFastCall InterlockedFlushSList, 1 cPublicFpo 0,1 ; ; Save nonvolatile registers and read the listhead sequence number followed ; by the listhead next link. ; ; N.B. These two dwords MUST be read exactly in this order. ; push ebx ; save nonvolatile registers push ebp ; mov ebp, ecx ; save listhead address mov edx, [ebp] + 4 ; get current sequence number mov eax, [ebp] + 0 ; get current next link ; ; N.B. The following code is the retry code should the compare ; part of the compare exchange operation fail ; ; If the list is empty, then there is nothing that can be removed. ; Efls10: or eax, eax ; check if list is empty jz short Efls20 ; if z set, list is empty mov ecx, 0 ; clear sequence number and depth mov ebx, 0 ; clear successor entry pointer .586 cmpxchg8b qword ptr [ebp] ; compare and exchange .386 jnz short Efls10 ; if z clear, exchange failed ; ; Restore nonvolatile registers and return result. ; cPublicFpo 0,0 Efls20: pop ebp ; restore nonvolatile registers pop ebx ; fstRET InterlockedFlushSList fstENDP InterlockedFlushSList page , 132 subttl "Interlocked i386 Exchange Ulong" ;++ ; ; ULONG ; FASTCALL ; InterlockedExchange ( ; IN PULONG Target, ; IN ULONG Value ; ) ; ; Routine Description: ; ; This function atomically exchanges the Target and Value, returning ; the prior contents of Target ; ; See Exi386InterlockedExchangeUlong. This function is the i386 ; architectural specific version of ExInterlockedDecrementLong. ; No source directly calls this function, instead ; ExInterlockedDecrementLong is called and when built on x86 these ; calls are macroed to the i386 optimized version. ; ; Arguments: ; ; (ecx) = Source - Address of ULONG to exchange ; (edx) = Value - New value of ULONG ; ; Return Value: ; ; The prior value of Source ;-- cPublicFastCall InterlockedExchange, 2 cPublicFpo 0,0 .486 mov eax, [ecx] ; get comperand value Ixchg: cmpxchg [ecx], edx ; compare and swap jnz Ixchg ; if nz, exchange failed .386 fstRET InterlockedExchange fstENDP InterlockedExchange ;++ ; ; LONG ; InterlockedIncrement( ; IN PLONG Addend ; ) ; ; Routine Description: ; ; This function performs an interlocked add of one to the addend variable. ; ; No checking is done for overflow. ; ; Arguments: ; ; Addend (ecx) - Supplies a pointer to a variable whose value is to be ; incremented by one. ; ; Return Value: ; ; (eax) - The incremented value. ; ;-- cPublicFastCall InterlockedIncrement,1 cPublicFpo 0,0 mov eax, 1 ; set increment value .486 xadd [ecx], eax ; interlocked increment .386p inc eax ; adjust return value fstRET InterlockedIncrement fstENDP InterlockedIncrement page , 132 subttl "InterlockedDecrment" ;++ ; ; LONG ; InterlockedDecrement( ; IN PLONG Addend ; ) ; ; Routine Description: ; ; This function performs an interlocked add of -1 to the addend variable. ; ; No checking is done for overflow ; ; Arguments: ; ; Addend (ecx) - Supplies a pointer to a variable whose value is to be ; decremented by one. ; ; Return Value: ; ; (eax) - The decremented value. ; ;-- cPublicFastCall InterlockedDecrement,1 cPublicFpo 0,0 mov eax, -1 ; set decrment value .486 xadd [ecx], eax ; interlocked decrement .386 dec eax ; adjust return value fstRET InterlockedDecrement fstENDP InterlockedDecrement page , 132 subttl "Interlocked Compare Exchange" ;++ ; ; PVOID ; FASTCALL ; InterlockedCompareExchange ( ; IN OUT PVOID *Destination, ; IN PVOID Exchange, ; IN PVOID Comperand ; ) ; ; Routine Description: ; ; This function performs an interlocked compare of the destination ; value with the comperand value. If the destination value is equal ; to the comperand value, then the exchange value is stored in the ; destination. Otherwise, no operation is performed. ; ; Arguments: ; ; (ecx) Destination - Supplies a pointer to destination value. ; ; (edx) Exchange - Supplies the exchange value. ; ; [esp + 4] Comperand - Supplies the comperand value. ; ; Return Value: ; ; The initial destination value is returned as the function value. ; ;-- cPublicFastCall InterlockedCompareExchange, 3 cPublicFpo 0,0 mov eax, [esp + 4] ; set comperand value .486 cmpxchg [ecx], edx ; compare and exchange .386 fstRET InterlockedCompareExchange fstENDP InterlockedCompareExchange page , 132 subttl "Interlocked Compare Exchange 64-bits" ;++ ; ; LONGLONG ; FASTCALL ; ExInterlockedCompareExchange64 ( ; IN PLONGLONG Destination, ; IN PLONGLONG Exchange, ; IN PLONGLONG Comperand ; ) ; ; Routine Description: ; ; This function performs a compare and exchange of 64-bits. ; ; N.B. The cmpxchg8b instruction is only supported on some processors. ; If the host processor does not support this instruction, then ; then following code is patched to contain a jump to the normal ; compare exchange 64-bit code which has a compatible calling ; sequence and data structure. ; ; Arguments: ; ; (ecx) Destination - Supplies a pointer to the destination variable. ; ; (edx) Exchange - Supplies a pointer to the exchange value. ; ; (esp+4) Comperand - Supplies a pointer to the comperand value. ; ; Return Value: ; ; The current destination value is returned as the function value. ; ;-- cPublicFastCall ExInterlockedCompareExchange64, 3 cPublicFpo 0,1 ; ; Save nonvolatile registers and read the exchange and comperand values. ; push ebx ; save nonvolatile registers push ebp ; mov ebp, ecx ; set destination address mov ebx, [edx] ; get exchange value mov ecx, [edx] + 4 ; mov edx, [esp] + 12 ; get comperand address mov eax, [edx] ; get comperand value mov edx, [edx] + 4 ; .586 cmpxchg8b qword ptr[ebp] ; compare and exchange .386 ; ; Restore nonvolatile registers and return result in edx:eax. ; cPublicFpo 0,0 pop ebp ; restore nonvolatile registers pop ebx ; fstRET ExInterlockedCompareExchange64 fstENDP ExInterlockedCompareExchange64 page , 132 subttl "Interlocked Exchange Add" ;++ ; ; LONG ; FASTCALL ; InterlockedExchangeAdd ( ; IN OUT PLONG Addend, ; IN LONG Increment ; ) ; ; Routine Description: ; ; This function performs an interlocked add of an increment value to an ; addend variable of type unsinged long. The initial value of the addend ; variable is returned as the function value. ; ; It is NOT possible to mix ExInterlockedDecrementLong and ; ExInterlockedIncrementong with ExInterlockedAddUlong. ; ; ; Arguments: ; ; (ecx) Addend - Supplies a pointer to a variable whose value is to be ; adjusted by the increment value. ; ; (edx) Increment - Supplies the increment value to be added to the ; addend variable. ; ; Return Value: ; ; The initial value of the addend variable. ; ;-- cPublicFastCall InterlockedExchangeAdd, 2 cPublicFpo 0,0 .486 xadd [ecx], edx ; exchange add .386 mov eax, edx ; set initial value fstRET InterlockedExchangeAdd fstENDP InterlockedExchangeAdd _TEXT ends end