title "Context Swap" ; Copyright (c) 1989 Microsoft Corporation ; Module Name: ; ctxswap.asm ; Abstract: ; This module implements the code necessary to field the dispatch ; interrupt and to perform kernel initiated context switching. ; Author: ; Shie-Lin Tzong (shielint) 14-Jan-1990 ; Environment: ; Kernel mode only. ; Revision History: ; 22-feb-90 bryanwi ; write actual swap context procedure .486p .xlist include ks386.inc include i386\kimacro.inc include mac386.inc include callconv.inc .list EXTRNP KiAcquireQueuedSpinLock,1,,FASTCALL EXTRNP KiReleaseQueuedSpinLock,1,,FASTCALL EXTRNP KiTryToAcquireQueuedSpinLock,1,,FASTCALL EXTRNP KfAcquireSpinLock,1,IMPORT,FASTCALL EXTRNP HalClearSoftwareInterrupt,1,IMPORT,FASTCALL EXTRNP HalRequestSoftwareInterrupt,1,IMPORT,FASTCALL EXTRNP KiActivateWaiterQueue,1,,FASTCALL EXTRNP KiReadyThread,1,,FASTCALL EXTRNP KiWaitTest,2,,FASTCALL EXTRNP KfLowerIrql,1,IMPORT,FASTCALL EXTRNP KfRaiseIrql,1,IMPORT,FASTCALL EXTRNP _KeGetCurrentIrql,0,IMPORT EXTRNP _KeGetCurrentThread,0 EXTRNP _KiDeliverApc,3 EXTRNP _KiQuantumEnd,0 EXTRNP _KeBugCheckEx,5 EXTRNP _KeBugCheck,1 extrn _KiTrap13:PROC extrn KiRetireDpcList:PROC extrn _KeI386FxsrPresent:BYTE extrn _KiDispatcherLock:DWORD extrn _KeFeatureBits:DWORD extrn _KeThreadSwitchCounters:DWORD extrn _KeTickCount:DWORD extrn __imp_@KfLowerIrql@4:DWORD extrn _KiWaitInListHead:DWORD extrn _KiWaitOutListHead:DWORD extrn _KiDispatcherReadyListHead:DWORD extrn _KiIdleSummary:DWORD extrn _KiReadySummary:DWORD extrn _KiSwapContextNotifyRoutine:DWORD extrn _KiThreadSelectNotifyRoutine:DWORD if DBG extrn _KdDebuggerEnabled:BYTE EXTRNP _DbgBreakPoint,0 extrn _DbgPrint:near extrn _MsgDpcTrashedEsp:BYTE extrn _MsgDpcTimeout:BYTE extrn _KiDPCTimeout:DWORD endif _TEXT$00 SEGMENT PARA PUBLIC 'CODE' ASSUME DS:FLAT, ES:FLAT, SS:NOTHING, FS:NOTHING, GS:NOTHING page ,132 subttl "Unlock Dispatcher Database" ; VOID ; KiUnlockDispatcherDatabase ( ; IN KIRQL OldIrql ; ) ; Routine Description: ; This routine is entered at IRQL DISPATCH_LEVEL with the dispatcher ; database locked. Its function is to either unlock the dispatcher ; database and return or initiate a context switch if another thread ; has been selected for execution. ; Arguments: ; (TOS) Return address ; (ecx) OldIrql - Supplies the IRQL when the dispatcher database ; lock was acquired. ; Return Value: ; None. cPublicFastCall KiUnlockDispatcherDatabase, 1 ; Check if a new thread is scheduled for execution. cmp PCR[PcPrcbData+PbNextThread], 0 ; check if next thread jne short Kiu20 ; if ne, new thread scheduled ; Release dispatcher database lock, lower IRQL to its previous level, ; and return. Kiu00: ; ifndef NT_UP mov eax, PCR[PcPrcb] ; get address of PRCB push ecx ; save IRQL for lower IRQL lea ecx, [eax]+PbLockQueue+(8*LockQueueDispatcherLock) fstCall KiReleaseQueuedSpinLock pop ecx ; get OldIrql endif ; N.B. This exit jumps directly to the lower IRQL routine which has a ; compatible fastcall interface. jmp dword ptr [__imp_@KfLowerIrql@4] ; lower IRQL to previous level ; A new thread has been selected to run on the current processor, but ; the new IRQL is not below dispatch level. If the current processor is ; not executing a DPC, then request a dispatch interrupt on the current ; processor. Kiu10: cmp dword ptr PCR[PcPrcbData.PbDpcRoutineActive],0 ; check if DPC routine active jne short Kiu00 ; if ne, DPC routine is active push ecx ; save new IRQL ifndef NT_UP mov eax, PCR[PcPrcb] ; get address of PRCB lea ecx, [eax]+PbLockQueue+(8*LockQueueDispatcherLock) fstCall KiReleaseQueuedSpinLock endif mov cl, DISPATCH_LEVEL ; request dispatch interrupt fstCall HalRequestSoftwareInterrupt ; pop ecx ; restore new IRQL ; N.B. This exit jumps directly to the lower IRQL routine which has a ; compatible fastcall interface. jmp dword ptr [__imp_@KfLowerIrql@4] ; lower IRQL to previous level ; Check if the previous IRQL is less than dispatch level. Kiu20: cmp cl, DISPATCH_LEVEL ; check if IRQL below dispatch level jge short Kiu10 ; if ge, not below dispatch level ; There is a new thread scheduled for execution and the previous IRQL is ; less than dispatch level. Context switch to the new thread immediately. ; N.B. The following registers MUST be saved such that ebp is saved last. ; This is done so the debugger can find the saved ebp for a thread ; that is not currently in the running state. .fpo (0, 0, 0, 4, 1, 0) sub esp, 4*4 mov [esp+12], ebx ; save registers mov [esp+8], esi ; mov [esp+4], edi ; mov [esp+0], ebp ; mov ebx, PCR[PcSelfPcr] ; get address of PCR mov esi, [ebx].PcPrcbData.PbNextThread ; get next thread address mov edi, [ebx].PcPrcbData.PbCurrentThread ; get current thread address mov dword ptr [ebx].PcPrcbData.PbNextThread, 0 ; clear next thread address mov [ebx].PcPrcbData.PbCurrentThread, esi ; set current thread address mov [edi].ThWaitIrql, cl ; save previous IRQL mov ecx, edi ; set address of current thread fstCall KiReadyThread ; reready thread for execution mov cl, [edi].ThWaitIrql ; set APC interrupt bypass disable call SwapContext ; swap context or al, al ; check if kernel APC pending mov cl, [esi].ThWaitIrql ; get original wait IRQL jnz short Kiu50 ; if nz, kernel APC pending Kiu30: mov ebp, [esp+0] ; restore registers mov edi, [esp+4] ; mov esi, [esp+8] ; mov ebx, [esp+12] ; add esp, 4*4 ; N.B. This exit jumps directly to the lower IRQL routine which has a ; compatible fastcall interface. jmp dword ptr [__imp_@KfLowerIrql@4] ; lower IRQL to previous level Kiu50: mov cl, APC_LEVEL ; lower IRQL to APC level fstCall KfLowerIrql ; xor eax, eax ; set previous mode to kernel stdCall _KiDeliverApc, ; deliver kernel mode APC inc dword ptr [ebx].PcPrcbData.PbApcBypassCount ; increment count xor ecx, ecx ; set original wait IRQL jmp short Kiu30 fstENDP KiUnlockDispatcherDatabase page ,132 subttl "Swap Thread" ; VOID ; KiSwapThread ( ; VOID ; ) ; Routine Description: ; This routine is called to select the next thread to run on the ; current processor and to perform a context switch to the thread. ; Arguments: ; None. ; Return Value: ; Wait completion status (eax). cPublicFastCall KiSwapThread, 0 .fpo (0, 0, 0, 4, 1, 0) ; N.B. The following registers MUST be saved such that ebp is saved last. ; This is done so the debugger can find the saved ebp for a thread ; that is not currently in the running state. sub esp, 4*4 mov [esp+12], ebx ; save registers mov [esp+8], esi ; mov [esp+4], edi ; mov [esp+0], ebp ; mov ebx, PCR[PcSelfPcr] ; get address of PCR mov edx, [ebx].PcPrcbData.PbNextThread ; get next thread address or edx, edx ; check if next thread selected jnz Swt140 ; if nz, next thread selected ; Find the highest nibble in the ready summary that contains a set bit ; and left justify so the nibble is in bits <31:28> mov ecx, 16 ; set base bit number mov edi, _KiReadySummary ; get ready summary mov esi, edi ; copy ready summary shr esi, 16 ; isolate bits <31:16> of summary jnz short Swt10 ; if nz, bits <31:16> are nonzero xor ecx, ecx ; set base bit number mov esi, edi ; set bits <15:0> of summary Swt10: shr esi, 8 ; isolate bits <15:8> of low bits jz short Swt20 ; if z, bits <15:8> are zero add ecx, 8 ; add offset to nonzero byte Swt20: mov esi, edi ; isolate highest nonzero byte shr esi, cl ; add ecx, 3 ; adjust to high bit of nibble cmp esi, 10h ; check if high nibble nonzero jb short Swt30 ; if b, then high nibble is zero add ecx, 4 ; compute ready queue priority Swt30: mov esi, ecx ; left justify ready summary nibble not ecx ; shl edi, cl ; or edi, edi ; ; If the next bit is set in the ready summary, then scan the corresponding ; dispatcher ready queue. Swt40: js short Swt60 ; if s, queue contains an entry Swt50: sub esi, 1 ; decrement ready queue priority shl edi, 1 ; position next ready summary bit jnz short Swt40 ; if nz, more queues to scan ; All ready queues were scanned without finding a runnable thread so ; default to the idle thread and set the appropriate bit in idle summary. ifdef _COLLECT_SWITCH_DATA_ inc _KeThreadSwitchCounters + TwSwitchToIdle ; increment counter endif ifdef NT_UP mov _KiIdleSummary, 1 ; set idle summary bit else mov eax, [ebx].PcPrcbData.PbSetMember ; get processor set member or _KiIdleSummary, eax ; set idle summary bit endif mov edx, [ebx].PcPrcbData.PbIdleThread ; set idle thread address jmp Swt140 ; ; If the thread can execute on the current processor, then remove it from ; the dispatcher ready queue. align 4 swt60: lea ebp, [esi*8] + _KiDispatcherReadyListHead ; get ready queue address mov ecx, [ebp].LsFlink ; get address of first queue entry Swt70: mov edx, ecx ; compute address of thread object sub edx, ThWaitListEntry ; ifndef NT_UP mov eax, [edx].ThAffinity ; get thread affinity test eax, [ebx].PcPrcbData.PbSetMember ; test if compatible affinity jnz short Swt80 ; if nz, thread affinity compatible mov ecx, [ecx].LsFlink ; get address of next entry cmp ebp, ecx ; check if end of list jnz short Swt70 ; if nz, not end of list jmp short Swt50 ; ; If the thread last ran on the current processor, has been waiting for ; longer than a quantum, or its priority is greater than low realtime ; plus 9, then select the thread. Otherwise, an attempt is made to find ; a more appropriate candidate. align 4 Swt80: cmp _KiThreadSelectNotifyRoutine, 0 ; check for callout routine je short Swt85 ; if eq, no callout routine registered push edx ; save volatile registers push ecx ; mov ecx, [edx].EtCid.CidUniqueThread ; set trial thread unique id call [_KiThreadSelectNotifyRoutine] ; notify callout routine pop ecx ; restore volatile registers pop edx ; or eax, eax ; check if trial thread selectable jnz Swt120 ; if nz, trial thread selectable jmp Swt87 ; align 4 Swt85: mov al, [edx].ThNextProcessor ; get last processor number cmp al, [ebx].PcPrcbData.PbNumber ; check if current processor jz Swt120 ; if z, same as current processor mov al, [edx].ThIdealProcessor ; get ideal processor number cmp al, [ebx].PcPrcbData.PbNumber ; check if current processor jz short Swt120 ; if z, same as current processor Swt87: cmp esi, LOW_REALTIME_PRIORITY + 9 ; check if priority in range jae short Swt120 ; if ae, priority not in range mov edi, _KeTickCount + 0 ; get low part of tick count sub edi, [edx].ThWaitTime ; compute length of wait cmp edi, READY_SKIP_QUANTUM + 1 ; check if wait time exceeded jae short Swt120 ; if ae, wait time exceeded mov edi, edx ; set address of thread ; Search forward in the ready queue until the end of the list is reached ; or a more appropriate thread is found. Swt90: mov edi, [edi].ThWaitListEntry ; get address of next entry cmp ebp, edi ; check if end of list jz short Swt120 ; if z, end of list sub edi, ThWaitListEntry ; compute address of thread mov eax, [edi].ThAffinity ; get thread affinity test eax, [ebx].PcPrcbData.PbSetMember ; test if compatible infinity jz short Swt100 ; if z, thread affinity not compatible cmp _KiThreadSelectNotifyRoutine, 0 ; check for callout routine je short Swt95 ; if eq, no callout routine registered push edx ; save volatile registers push ecx ; mov ecx, [edi].EtCid.CidUniqueThread ; set trial thread unique id call [_KiThreadSelectNotifyRoutine] ; notify callout routine pop ecx ; restore volatile registers pop edx ; or eax, eax ; check if trial thread selectable jnz short Swt110 ; if nz, trial thread selectable jmp short Swt100 ; align 4 Swt95: mov al, [edi].ThNextProcessor ; get last processor number cmp al, [ebx].PcPrcbData.PbNumber ; check if current processor jz short Swt110 ; if z, same as current processor mov al, [edi].ThIdealProcessor ; get ideal processor number cmp al, [ebx].PcPrcbData.PbNumber ; check if current processor jz short Swt110 ; if z, same as current processor Swt100: mov eax, _KeTickCount + 0 ; get low part of tick count sub eax, [edi].ThWaitTime ; compute length of wait cmp eax, READY_SKIP_QUANTUM + 1 ; check if wait time exceeded jb short Swt90 ; if b, wait time not exceeded jmp short Swt120 ; align 4 Swt110: mov edx, edi ; set address of thread mov ecx, edi ; compute address of list entry add ecx, ThWaitListEntry ; Swt120: mov al, [ebx].PcPrcbData.PbNumber ; get current processor number ifdef _COLLECT_SWITCH_DATA_ lea ebp, _KeThreadSwitchCounters + TwFindIdeal ; get counter address cmp al, [edx].ThIdealProcessor ; check if same as ideal processor jz short Swt130 ; if z, same as ideal processor add ebp, TwFindLast - TwFindIdeal ; compute address of last counter cmp al, [edx].ThNextProcessor ; check if same as last processor jz short Swt130 ; if z, same as last processor add ebp,TwFindAny - TwFindLast ; compute address of correct counter Swt130: inc dword ptr [ebp] ; increment appropriate switch counter endif mov [edx].ThNextProcessor, al ; set next processor number endif ; Remove the selected thread from the ready queue. mov eax, [ecx].LsFlink ; get list entry forward link mov ebp, [ecx].LsBlink ; get list entry backward link mov [ebp].LsFlink, eax ; set forward link in previous entry mov [eax].LsBlink, ebp ; set backward link in next entry cmp eax, ebp ; check if list is empty jnz short Swt140 ; if nz, list is not empty mov ebp, 1 ; clear ready summary bit mov ecx, esi ; shl ebp, cl ; xor _KiReadySummary, ebp ; ; Swap context to the next thread. Swt140: mov esi, edx ; set address of next thread mov edi, [ebx].PcPrcbData.PbCurrentThread ; set current thread address mov dword ptr [ebx].PcPrcbData.PbNextThread, 0 ; clear next thread address mov [ebx].PcPrcbData.PbCurrentThread, esi ; set current thread address mov cl, [edi].ThWaitIrql ; set APC interrupt bypass disable call SwapContext ; swap context or al, al ; check if kernel APC pending mov edi, [esi].ThWaitStatus ; save wait completion status mov cl, [esi].ThWaitIrql ; get wait IRQL jnz short Swt160 ; if nz, kernel APC pending Swt150: fstCall KfLowerIrql ; lower IRQL to previous value mov eax, edi ; set wait completion status mov ebp, [esp+0] ; restore registers mov edi, [esp+4] ; mov esi, [esp+8] ; mov ebx, [esp+12] ; add esp, 4*4 ; fstRET KiSwapThread ; Swt160: mov cl, APC_LEVEL ; lower IRQL to APC level fstCall KfLowerIrql ; xor eax, eax ; set previous mode to kernel stdCall _KiDeliverApc, ; deliver kernel mode APC inc dword ptr [ebx].PcPrcbData.PbApcBypassCount ; increment count xor ecx, ecx ; set original wait IRQL jmp short Swt150 fstENDP KiSwapThread page ,132 subttl "Dispatch Interrupt" ; Routine Description: ; This routine is entered as the result of a software interrupt generated ; at DISPATCH_LEVEL. Its function is to process the Deferred Procedure Call ; (DPC) list, and then perform a context switch if a new thread has been ; selected for execution on the processor. ; This routine is entered at IRQL DISPATCH_LEVEL with the dispatcher ; database unlocked. When a return to the caller finally occurs, the ; IRQL remains at DISPATCH_LEVEL, and the dispatcher database is still ; unlocked. ; Arguments: ; None ; Return Value: ; None. align 16 cPublicProc _KiDispatchInterrupt ,0 cPublicFpo 0, 0 mov ebx, PCR[PcSelfPcr] ; get address of PCR kdi00: lea eax, [ebx].PcPrcbData.PbDpcListHead ; get DPC listhead address ; Disable interrupts and check if there is any work in the DPC list ; of the current processor. kdi10: cli ; disable interrupts cmp eax, [eax].LsFlink ; check if DPC List is empty je short kdi40 ; if eq, list is empty push ebp ; save register ; Exceptions occuring in DPCs are unrelated to any exception handlers ; in the interrupted thread. Terminate the exception list. push [ebx].PcExceptionList mov [ebx].PcExceptionList, EXCEPTION_CHAIN_END ; Switch to the DPC stack for this processor. mov edx, esp mov esp, [ebx].PcPrcbData.PbDpcStack push edx .fpo (0, 0, 0, 1, 1, 0) mov ebp, eax ; set address of DPC listhead call KiRetireDpcList ; process the current DPC list ; Switch back to the current thread stack, restore the exception list ; and saved EBP. pop esp pop [ebx].PcExceptionList pop ebp .fpo (0, 0, 0, 0, 0, 0) ; Check to determine if quantum end is requested. ; N.B. If a new thread is selected as a result of processing the quantum ; end request, then the new thread is returned with the dispatcher ; database locked. Otherwise, NULL is returned with the dispatcher ; database unlocked. kdi40: sti ; enable interrupts cmp dword ptr [ebx].PcPrcbData.PbQuantumEnd, 0 ; quantum end requested jne kdi90 ; if neq, quantum end request ; Check to determine if a new thread has been selected for execution on this ; processor. cmp dword ptr [ebx].PcPrcbData.PbNextThread, 0 ; check addr of next thread object je short kdi70 ; if eq, then no new thread ; Disable interrupts and attempt to acquire the dispatcher database lock. ifndef NT_UP cli cmp dword ptr _KiDispatcherLock, 0 jnz short kdi80 lea ecx, [ebx]+PcPrcbData+PbLockQueue+(8*LockQueueDispatcherLock) fstCall KiTryToAcquireQueuedSpinLock jz short kdi80 ; jif not acquired mov ecx, SYNCH_LEVEL ; raise IRQL to synchronization level fstCall KfRaiseIrql ; sti ; enable interrupts endif mov eax, [ebx].PcPrcbData.PbNextThread ; get next thread address ; N.B. The following registers MUST be saved such that ebp is saved last. ; This is done so the debugger can find the saved ebp for a thread ; that is not currently in the running state. .fpo (0, 0, 0, 3, 1, 0) kdi60: sub esp, 3*4 mov [esp+8], esi ; save registers mov [esp+4], edi ; mov [esp+0], ebp ; mov esi, eax ; set next thread address mov edi, [ebx].PcPrcbData.PbCurrentThread ; get current thread address mov dword ptr [ebx].PcPrcbData.PbNextThread, 0 ; clear next thread address mov [ebx].PcPrcbData.PbCurrentThread, esi ; set current thread address mov ecx, edi ; set address of current thread fstCall KiReadyThread ; ready thread (ecx) for execution mov cl, 1 ; set APC interrupt bypass disable call SwapContext ; call context swap routine mov ebp, [esp+0] ; restore registers mov edi, [esp+4] ; mov esi, [esp+8] ; add esp, 3*4 kdi70: stdRET _KiDispatchInterrupt ; return ; Enable interrupts and check DPC queue. ifndef NT_UP kdi80: sti ; enable interrupts YIELD ; rest jmp kdi00 ; endif ; Process quantum end event. ; N.B. If the quantum end code returns a NULL value, then no next thread ; has been selected for execution. Otherwise, a next thread has been ; selected and the dispatcher databased is locked. kdi90: mov dword ptr [ebx].PcPrcbData.PbQuantumEnd, 0 ; clear quantum end indicator stdCall _KiQuantumEnd ; process quantum end or eax, eax ; check if new thread selected jne short kdi60 ; if ne, new thread selected stdRET _KiDispatchInterrupt ; return stdENDP _KiDispatchInterrupt page ,132 subttl "Swap Context to Next Thread" ; Routine Description: ; This routine is called to swap context from one thread to the next. ; It swaps context, flushes the data, instruction, and translation ; buffer caches, restores nonvolatile integer registers, and returns ; to its caller. ; N.B. It is assumed that the caller (only callers are within this ; module) saved the nonvolatile registers, ebx, esi, edi, and ; ebp. This enables the caller to have more registers available. ; Arguments: ; cl - APC interrupt bypass disable (zero enable, nonzero disable). ; edi - Address of previous thread. ; esi - Address of next thread. ; ebx - Address of PCR. ; Return value: ; al - Kernel APC pending. ; ebx - Address of PCR. ; esi - Address of current thread object. align 16 public SwapContext ifndef NT_UP public _ScPatchFxb public _ScPatchFxe endif SwapContext proc ; NOTE: The ES: override on the move to ThState is part of the ; lazy-segment load system. It assures that ES has a valid ; selector in it, thus preventing us from propagating a bad ; ES accross a context switch. ; Note that if segments, other than the standard flat segments, ; with limits above 2 gig exist, neither this nor the rest of ; lazy segment loads are reliable. ; Note that ThState must be set before the dispatcher lock is released ; to prevent KiSetPriorityThread from seeing a stale value. ; Save the APC disable flag and set new thread state to running. or cl, cl ; set zf in flags mov byte ptr es:[esi]+ThState, Running ; set thread state to running pushfd cPublicFpo 0, 1 ; Acquire the context swap lock so the address space of the old process ; cannot be deleted and then release the dispatcher database lock. ; N.B. This lock is used to protect the address space until the context ; switch has sufficiently progressed to the point where the address ; space is no longer needed. This lock is also acquired by the reaper ; thread before it finishes thread termination. ifndef NT_UP lea ecx, [ebx]+PcPrcbData+PbLockQueue+(8*LockQueueContextSwapLock) fstCall KiAcquireQueuedSpinLock ; Acquire ContextSwap lock lea ecx, [ebx]+PcPrcbData+PbLockQueue+(8*LockQueueDispatcherLock) fstCall KiReleaseQueuedSpinLock ; release Dispatcher Lock endif ; Save the APC disable flag and the exception listhead. ; (also, check for DPC running which is illegal right now). mov ecx, [ebx]+PcExceptionList ; save exception list cmp [ebx]+PcPrcbData+PbDpcRoutineActive, 0 push ecx cPublicFpo 0, 2 jne sc91 ; bugcheck if DPC active. ; Notify registered callout routine of swap context. ifndef NT_UP cmp _KiSwapContextNotifyRoutine, 0 ; check for callout routine jne sc92 ; if ne, callout routine registered sc03: endif ; Accumulate the total time spent in a thread. ifdef PERF_DATA test _KeFeatureBits, KF_RDTSC ; feature supported? jz short @f ; if z, feature not present .586p rdtsc ; read cycle counter .486p sub eax, [ebx].PcPrcbData.PbThreadStartCount.LiLowPart ; sub off thread sbb edx, [ebx].PcPrcbData.PbThreadStartCount.LiHighPart ; starting time add [edi].EtPerformanceCountLow, eax ; accumlate thread run time adc [edi].EtPerformanceCountHigh, edx ; add [ebx].PcPrcbData.PbThreadStartCount.LiLowPart, eax ; set new thread adc [ebx].PcPrcbData.PbThreadStartCount.LiHighPart, edx ; starting time @@: ; endif ; On a uniprocessor system the NPX state is swapped in a lazy manner. ; If a thread whose state is not in the coprocessor attempts to perform ; a coprocessor operation, the current NPX state is swapped out (if needed), ; and the new state is swapped in durning the fault. (KiTrap07) ; On a multiprocessor system we still fault in the NPX state on demand, but ; we save the state when the thread switches out (assuming the NPX state ; was loaded). This is because it could be difficult to obtain the thread's ; NPX in the trap handler if it was loaded into a different processor's ; coprocessor. mov ebp, cr0 ; get current CR0 mov edx, ebp ifndef NT_UP cmp byte ptr [edi]+ThNpxState, NPX_STATE_LOADED ; check if NPX state je sc_save_npx_state endif sc05: mov cl, [esi]+ThDebugActive ; get debugger active state mov [ebx]+PcDebugActive, cl ; set new debugger active state ; Switch stacks: ; 1. Save old esp in old thread object. ; 2. Copy stack base and stack limit into TSS AND PCR ; 3. Load esp from new thread object ; Keep interrupts off so we don't confuse the trap handler into thinking ; we've overrun the kernel stack. cli ; disable interrupts mov [edi]+ThKernelStack, esp ; save old kernel stack pointer mov eax, [esi]+ThInitialStack ; get new initial stack pointer mov ecx, [esi]+ThStackLimit ; get stack limit sub eax, NPX_FRAME_LENGTH ; space for NPX_FRAME & NPX CR0 flags mov [ebx]+PcStackLimit, ecx ; set new stack limit mov [ebx]+PcInitialStack, eax ; set new stack base .errnz (NPX_STATE_NOT_LOADED - CR0_TS - CR0_MP) .errnz (NPX_STATE_LOADED - 0) ; (eax) = Initial Stack ; (ebx) = Prcb ; (edi) = OldThread ; (esi) = NewThread ; (ebp) = Current CR0 ; (edx) = Current CR0 xor ecx, ecx mov cl, [esi]+ThNpxState ; New NPX state is (or is not) loaded and edx, NOT (CR0_MP+CR0_EM+CR0_TS) ; clear thread settable NPX bits or ecx, edx ; or in new thread's cr0 or ecx, [eax]+FpCr0NpxState ; merge new thread settable state cmp ebp, ecx ; check if old and new CR0 match jne sc_reload_cr0 ; if ne, no change in CR0 ; N.B. It is important that the following adjustment NOT be applied to ; the initial stack value in the PCR. If it is, it will cause the ; location in memory that the processor pushes the V86 mode segment ; registers and the first 4 ULONGs in the FLOATING_SAVE_AREA to ; occupy the same memory locations, which could result in either ; trashed segment registers in V86 mode, or a trashed NPX state. ; Adjust ESP0 so that V86 mode threads and 32 bit threads can share ; a trapframe structure, and the NPX save area will be accessible ; in the same manner on all threads ; This test will check the user mode flags. On threads with no user ; mode context, the value of esp0 does not matter (we will never run ; in user mode without a usermode context, and if we don't run in user ; mode the processor will never use the esp0 value. align 4 sc06: test dword ptr [eax] - KTRAP_FRAME_LENGTH + TsEFlags, EFLAGS_V86_MASK jnz short sc07 ; if nz, V86 frame, no adjustment sub eax, TsV86Gs - TsHardwareSegSs ; bias for missing fields sc07: mov ecx, [ebx]+PcTss ; mov [ecx]+TssEsp0, eax ; mov esp, [esi]+ThKernelStack ; set new stack pointer mov eax, [esi]+ThTeb ; get user TEB address mov [ebx]+PcTeb, eax ; set user TEB address ; Edit the TEB descriptor to point to the TEB sti ; enable interrupts mov ecx, [ebx]+PcGdt ; mov [ecx]+(KGDT_R3_TEB+KgdtBaseLow), ax ; shr eax, 16 ; mov [ecx]+(KGDT_R3_TEB+KgdtBaseMid), al ; shr eax, 8 mov [ecx]+(KGDT_R3_TEB+KgdtBaseHi), al ; NOTE: Keep KiSwapProcess (below) in sync with this code! ; If the new process is not the same as the old process, then switch the ; address space to the new process. mov eax, [edi].ThApcState.AsProcess ; get old process address cmp eax, [esi].ThApcState.AsProcess ; check if process match jz short sc22 ; if z, old and new process match mov edi, [esi].ThApcState.AsProcess ; get new process address ; Update the processor set masks. ifndef NT_UP if DBG mov cl, [esi]+ThNextProcessor ; get current processor number cmp cl, [ebx]+PcPrcbData+PbNumber ; same as running processor? jne sc_error2 ; if ne, processor number mismatch endif mov ecx, [ebx]+PcSetMember ; get processor set member xor [eax]+PrActiveProcessors, ecx ; clear bit in old processor set xor [edi]+PrActiveProcessors, ecx ; set bit in new processor set if DBG test [eax]+PrActiveProcessors, ecx ; test if bit clear in old set jnz sc_error4 ; if nz, bit not clear in old set test [edi]+PrActiveProcessors, ecx ; test if bit set in new set jz sc_error5 ; if z, bit not set in new set endif endif ; New CR3, flush tb, sync tss, set IOPM ; CS, SS, DS, ES all have flat (GDT) selectors in them. ; FS has the pcr selector. ; Therefore, GS is only selector we need to flush. We null it out, ; it will be reloaded from a stack frame somewhere above us. ; Note: this load of GS before CR3 works around P6 step B0 errata 11 xor eax, eax ; assume null ldt mov gs, ax ; mov eax, [edi]+PrDirectoryTableBase ; get new directory base mov ebp, [ebx]+PcTss ; get new TSS mov ecx, [edi]+PrIopmOffset ; get IOPM offset mov [ebp]+TssCR3, eax ; make TSS be in sync with hardware mov cr3, eax ; flush TLB and set new directory base mov [ebp]+TssIoMapBase, cx ; ; LDT switch xor eax, eax ; check if null LDT limit cmp word ptr [edi]+PrLdtDescriptor, ax jnz short sc_load_ldt ; if nz, LDT limit lldt ax ; set LDT ; Release the context swap lock. align 4 sc22: ; ifndef NT_UP lea ecx, [ebx]+PcPrcbData+PbLockQueue+(8*LockQueueContextSwapLock) fstCall KiReleaseQueuedSpinLock endif ; Update context switch counters. inc dword ptr [esi]+ThContextSwitches ; thread count inc dword ptr [ebx]+PcPrcbData+PbContextSwitches ; processor count pop ecx ; restore exception list mov [ebx].PcExceptionList, ecx ; ; If the new thread has a kernel mode APC pending, then request an APC ; interrupt. cmp byte ptr [esi].ThApcState.AsKernelApcPending, 0 ; APC pending? jne short sc80 ; if ne, kernel APC pending popfd ; restore flags xor eax, eax ; clear kernel APC pending ret ; return ; The new thread has an APC interrupt pending. If APC interrupt bypass is ; enable, then return kernel APC pending. Otherwise, request a software ; interrupt at APC_LEVEL and return no kernel APC pending. sc80: popfd ; restore flags jnz short sc90 ; if nz, APC interupt bypass disabled mov al, 1 ; set kernel APC pending ret ; sc90: mov cl, APC_LEVEL ; request software interrupt level fstCall HalRequestSoftwareInterrupt ; xor eax, eax ; clear kernel APC pending ret ; ; Set for new LDT value sc_load_ldt: mov ebp, [ebx]+PcGdt ; mov eax, [edi+PrLdtDescriptor] ; mov [ebp+KGDT_LDT], eax ; mov eax, [edi+PrLdtDescriptor+4] ; mov [ebp+KGDT_LDT+4], eax ; mov eax, KGDT_LDT ; ; Set up int 21 descriptor of IDT. If the process does not have an Ldt, it ; should never make any int 21 calls. If it does, an exception is generated. If ; the process has an Ldt, we need to update int21 entry of LDT for the process. ; Note the Int21Descriptor of the process may simply indicate an invalid ; entry. In which case, the int 21 will be trapped to the kernel. mov ebp, [ebx]+PcIdt ; mov ecx, [edi+PrInt21Descriptor] ; mov [ebp+21h*8], ecx ; mov ecx, [edi+PrInt21Descriptor+4] ; mov [ebp+21h*8+4], ecx ; lldt ax ; set LDT jmp short sc22 ; Cr0 has changed (ie, floating point processor present), load the new value. sc_reload_cr0: if DBG test byte ptr [esi]+ThNpxState, NOT (CR0_TS+CR0_MP) jnz sc_error ; test dword ptr [eax]+FpCr0NpxState, NOT (CR0_PE+CR0_MP+CR0_EM+CR0_TS) jnz sc_error3 ; endif mov cr0,ecx ; set new CR0 NPX state jmp sc06 ifndef NT_UP ; Save coprocessor's current context. FpCr0NpxState is the current thread's ; CR0 state. The following bits are valid: CR0_MP, CR0_EM, CR0_TS. MVDMs ; may set and clear MP & EM as they please and the settings will be reloaded ; on a context switch (but they will not be saved from CR0 to Cr0NpxState). ; The kernel sets and clears TS as required. ; (ebp) = Current CR0 ; (edx) = Current CR0 sc_save_npx_state: and edx, NOT (CR0_MP+CR0_EM+CR0_TS) ; we need access to the NPX state mov ecx,[ebx]+PcInitialStack ; get NPX save save area address cmp ebp, edx ; Does CR0 need reloading? je short sc_npx10 mov cr0, edx ; set new cr0 mov ebp, edx ; (ebp) = (edx) = current cr0 state sc_npx10: ; The fwait following the fnsave is to make sure that the fnsave has stored the ; data into the save area before this coprocessor state could possibly be ; context switched in and used on a different (co)processor. I've added the ; clocks from when the dispatcher lock is released and don't believe it's a ; possibility. I've also timed the impact this fwait seems to have on a 486 ; when performing lots of numeric calculations. It appears as if there is ; nothing to wait for after the fnsave (although the 486 manual says there is) ; and therefore the calculation time far outweighed the 3clk fwait and it ; didn't make a noticable difference. ; If FXSR feature is NOT present on the processor, the fxsave instruction is ; patched at boot time to start using fnsave instead _ScPatchFxb: ; fxsave [ecx] ; save NPX state db 0FH, 0AEH, 01 _ScPatchFxe: mov byte ptr [edi]+ThNpxState, NPX_STATE_NOT_LOADED ; set no NPX state mov dword ptr [ebx].PcPrcbData+PbNpxThread, 0 ; clear npx owner jmp sc05 ; Notify context swap callout routine. This code is out of line to ; optimize the normal case (which is expected to be the case where ; there is no callout routine. sc92: mov edx, [esi].EtCid.CidUniqueThread ; set new thread unique id mov ecx, [edi].EtCid.CidUniqueThread ; set old thread unique id call [_KiSwapContextNotifyRoutine] ; notify callout routine jmp sc03 endif .fpo (2, 0, 0, 0, 0, 0) sc91: stdCall _KeBugCheck ret ; return if DBG sc_error5: int 3 sc_error4: int 3 sc_error3: int 3 sc_error2: int 3 sc_error: int 3 endif SwapContext endp page , 132 subttl "Interlocked Swap PTE" ; HARDWARE_PTE ; KeInterlockedSwapPte ( ; IN PKI_INTERNAL_PTE PtePointer, ; IN PKI_INTERNAL_PTE NewPteContents ; ) ; Routine Description: ; This function performs an interlocked swap of a PTE. This is only needed ; for the PAE architecture where the PTE width is larger than the register ; width. ; Both PTEs must be valid or a careful write would have been done instead. ; Arguments: ; PtePointer - Address of Pte to update with new value. ; NewPteContents - Pointer to the new value to put in the Pte. Will simply ; be assigned to *PtePointer, in a fashion correct for the hardware. ; Return Value: ; Returns the contents of the PtePointer before the new value ; is stored. cPublicProc _KeInterlockedSwapPte ,2 push ebx push esi mov ebx, [esp] + 16 ; ebx = NewPteContents mov esi, [esp] + 12 ; esi = PtePointer mov ecx, [ebx] + 4 mov ebx, [ebx] ; ecx:ebx = source pte contents mov edx, [esi] + 4 mov eax, [esi] ; edx:eax = target pte contents swapagain: ; cmpxchg loads edx:eax with the new contents of the target quadword ; in the event of failure .586 ifndef NT_UP lock cmpxchg8b qword ptr [esi] ; compare and exchange else cmpxchg8b qword ptr [esi] ; compare and exchange endif .386 jnz short swapagain ; if z clear, exchange failed pop esi pop ebx stdRET _KeInterlockedSwapPte stdENDP _KeInterlockedSwapPte page , 132 subttl "Flush EntireTranslation Buffer" ; VOID ; KeFlushCurrentTb ( ; ) ; Routine Description: ; This function flushes the entire translation buffer (TB) on the current ; processor and also flushes the data cache if an entry in the translation ; buffer has become invalid. ; Arguments: ; Return Value: ; None. cPublicProc _KeFlushCurrentTb ,0 ktb00: mov eax, cr3 ; (eax) = directory table base mov cr3, eax ; flush TLB stdRET _KeFlushCurrentTb .586p ktb_gb: mov eax, cr4 ; *** see Ki386EnableGlobalPage *** and eax, not CR4_PGE ; This FlushCurrentTb version gets copied into mov cr4, eax ; ktb00 at initialization time if needed. or eax, CR4_PGE mov cr4, eax ktb_eb: stdRET _KeFlushCurrentTb .486p stdENDP _KeFlushCurrentTb ;; moved KiFlushDcache below KeFlushCurrentTb for BBT purposes. BBT ;; needs an end label to treat KeFlushCurrentTb as data and to keep together. page , 132 subttl "Flush Data Cache" ; VOID ; KiFlushDcache ( ; ) ; VOID ; KiFlushIcache ( ; ) ; Routine Description: ; This routine does nothing on i386 and i486 systems. Why? Because ; (a) their caches are completely transparent, (b) they don't have ; instructions to flush their caches. ; Arguments: ; None. ; Return Value: ; None. cPublicProc _KiFlushDcache ,0 cPublicProc _KiFlushIcache ,0 stdRET _KiFlushIcache stdENDP _KiFlushIcache stdENDP _KiFlushDcache _TEXT$00 ends INIT SEGMENT DWORD PUBLIC 'CODE' ASSUME DS:FLAT, ES:FLAT, SS:NOTHING, FS:NOTHING, GS:NOTHING ; VOID Ki386EnableGlobalPage (IN volatile PLONG Number) ; /*++ ; Routine Description: ; This routine enables the global page PDE/PTE support in the system, ; and stalls until complete and them sets the current processor's cr4 ; register to enable global page support. ; Arguments: ; Number - Supplies a pointer to the count of the number of processors in the configuration. ; Return Value: ; None. cPublicProc _Ki386EnableGlobalPage,1 push esi push edi push ebx mov edx, [esp+16] ; pointer to Number pushfd cli ; Wait for all processors lock dec dword ptr [edx] ; count down egp10: YIELD cmp dword ptr [edx], 0 ; wait for all processors to signal jnz short egp10 cmp PCR[PcNumber], 0 ; processor 0? jne short egp20 ; Install proper KeFlushCurrentTb function. mov edi, ktb00 mov esi, ktb_gb mov ecx, ktb_eb - ktb_gb + 1 rep movsb mov byte ptr [ktb_eb], 0 ; Wait for P0 to signal that proper flush TB handlers have been installed egp20: cmp byte ptr [ktb_eb], 0 jnz short egp20 ; Flush TB, and enable global page support ; (note load of CR4 is explicitly done before the load of CR3 ; to work around P6 step B0 errata 11) .586p mov eax, cr4 and eax, not CR4_PGE ; should not be set, but let's be safe mov ecx, cr3 mov cr4, eax mov cr3, ecx ; Flush TB or eax, CR4_PGE ; enable global TBs mov cr4, eax .486p popfd pop ebx pop edi pop esi stdRET _Ki386EnableGlobalPage stdENDP _Ki386EnableGlobalPage ; VOID ; Ki386EnableFxsr ( ; IN volatile PLONG Number ; ) ; /*++ ; Routine Description: ; This routine sets OSFXSR bit in CR4 to indicate that OS supports ; FXSAVE/FXRSTOR for use during context switches ; Arguments: ; Number - Supplies a pointer to the count of the number of processors in ; the configuration. ; Return Value: ; None. cPublicProc _Ki386EnableFxsr,1 .586p mov eax, cr4 or eax, CR4_FXSR mov cr4, eax .386p stdRET _Ki386EnableFxsr stdENDP _Ki386EnableFxsr ; VOID ; Ki386EnableXMMIExceptions ( ; IN volatile PLONG Number ; ) ; /*++ ; Routine Description: ; This routine installs int 19 XMMI unmasked Numeric Exception handler ; and sets OSXMMEXCPT bit in CR4 to indicate that OS supports ; unmasked Katmai New Instruction technology exceptions. ; Arguments: ; Number - Supplies a pointer to count of the number of processors in ; the configuration. ; Return Value: ; None. cPublicProc _Ki386EnableXMMIExceptions,1 ;Set up IDT for INT19 mov ecx,PCR[PcIdt] ;Get IDT address lea eax, [ecx] + 098h ;XMMI exception is int 19 mov byte ptr [eax + 5], 08eh ;P=1,DPL=0,Type=e mov word ptr [eax + 2], KGDT_R0_CODE ;Kernel code selector mov edx, offset FLAT:_KiTrap13 ;Address of int 19 handler mov ecx,edx mov word ptr [eax],cx ;addr moves into low byte shr ecx,16 mov word ptr [eax + 6],cx ;addr moves into high byte .586p ;Enable XMMI exception handling mov eax, cr4 or eax, CR4_XMMEXCPT mov cr4, eax .386p stdRET _Ki386EnableXMMIExceptions stdENDP _Ki386EnableXMMIExceptions ; VOID ; Ki386EnableCurrentLargePage ( ; IN ULONG IdentityAddr, ; IN ULONG IdentityCr3 ; ) ; /*++ ; Routine Description: ; This routine enables the large page PDE support in the processor. ; Arguments: ; IdentityAddr - Supplies the linear address of the beginning of this ; function where (linear == physical). ; IdentityCr3 - Supplies a pointer to the temporary page directory and ; page tables that provide both the kernel (virtual ->physical) and ; identity (linear->physical) mappings needed for this function. ; Return Value: ; None. public _Ki386EnableCurrentLargePageEnd cPublicProc _Ki386EnableCurrentLargePage,2 mov ecx,[esp]+4 ; (ecx)-> IdentityAddr mov edx,[esp]+8 ; (edx)-> IdentityCr3 pushfd ; save current IF state cli ; disable interrupts mov eax, cr3 ; (eax)-> original Cr3 mov cr3, edx ; load Cr3 with Identity mapping sub ecx, offset _Ki386EnableCurrentLargePage add ecx, offset _Ki386LargePageIdentityLabel jmp ecx ; jump to (linear == physical) _Ki386LargePageIdentityLabel: mov ecx, cr0 and ecx, NOT CR0_PG ; clear PG bit to disable paging mov cr0, ecx ; disable paging jmp $+2 .586p mov edx, cr4 or edx, CR4_PSE ; enable Page Size Extensions mov cr4, edx .486p mov edx, offset OriginalMapping or ecx, CR0_PG ; set PG bit to enable paging mov cr0, ecx ; enable paging jmp edx ; Return to original mapping. OriginalMapping: mov cr3, eax ; restore original Cr3 popfd ; restore interrupts to previous stdRET _Ki386EnableCurrentLargePage _Ki386EnableCurrentLargePageEnd: stdENDP _Ki386EnableCurrentLargePage INIT ends _TEXT$00 SEGMENT PARA PUBLIC 'CODE' ASSUME DS:FLAT, ES:FLAT, SS:NOTHING, FS:NOTHING, GS:NOTHING page , 132 subttl "Flush Single Translation Buffer" ; VOID ; FASTCALL ; KiFlushSingleTb ( ; IN BOOLEAN Invalid, ; IN PVOID Virtual ; ) ; Routine Description: ; This function flushes a single TB entry. ; It only works on a 486 or greater. ; Arguments: ; Invalid - Supplies a boolean value that specifies the reason for ; flushing the translation buffer. ; Virtual - Supplies the virtual address of the single entry that is ; to be flushed from the translation buffer. ; Return Value: ; None. cPublicFastCall KiFlushSingleTb ,2 ; 486 or above code invlpg [edx] fstRET KiFlushSingleTb fstENDP KiFlushSingleTb page , 132 subttl "Swap Process" ; VOID ; KiSwapProcess ( ; IN PKPROCESS NewProcess, ; IN PKPROCESS OldProcess ; ) ; Routine Description: ; This function swaps the address space to another process by flushing ; the data cache, the instruction cache, the translation buffer, and ; establishes a new directory table base. ; It also swaps in the LDT and IOPM of the new process. This is necessary ; to avoid bogus mismatches in SwapContext. ; NOTE: keep in sync with process switch part of SwapContext ; Arguments: ; Process - Supplies a pointer to a control object of type process. ; Return Value: ; None. cPublicProc _KiSwapProcess ,2 cPublicFpo 2, 0 ifndef NT_UP ; Acquire the context swap lock. mov ecx, PCR[PcPrcb] lea ecx, [ecx]+PbLockQueue+(8*LockQueueContextSwapLock) fstCall KiAcquireQueuedSpinLock mov edx,[esp]+4 ; (edx)-> New process mov eax,[esp]+8 ; (eax)-> Old Process ; Clear the processor set member in the old process, set the processor ; member in the new process, and release the context swap lock. mov ecx, PCR[PcSetMember] xor [eax]+PrActiveProcessors,ecx ; clear bit in old processor set xor [edx]+PrActiveProcessors,ecx ; set bit in new processor set if DBG test [eax]+PrActiveProcessors,ecx ; test if bit clear in old set jnz kisp_error ; if nz, bit not clear in old set test [edx]+PrActiveProcessors,ecx ; test if bit set in new set jz kisp_error1 ; if z, bit not set in new set endif mov ecx, PCR[PcPrcb] lea ecx, [ecx]+PbLockQueue+(8*LockQueueContextSwapLock) fstCall KiReleaseQueuedSpinLock endif mov ecx,PCR[PcTss] ; (ecx)-> TSS mov edx,[esp]+4 ; (edx)-> New process ; Change address space xor eax,eax ; assume ldtr is to be NULL mov gs,ax ; Clear gs. (also workarounds ; P6 step B0 errata 11) mov eax,[edx]+PrDirectoryTableBase mov [ecx]+TssCR3,eax ; be sure TSS in sync with processor mov cr3,eax ; Change IOPM mov ax,[edx]+PrIopmOffset mov [ecx]+TssIoMapBase,ax ; Change LDT xor eax, eax cmp word ptr [edx]+PrLdtDescriptor,ax ; limit 0? jz short kisp10 ; null LDT, go load NULL ldtr ; Edit LDT descriptor mov ecx,PCR[PcGdt] mov eax,[edx+PrLdtDescriptor] mov [ecx+KGDT_LDT],eax mov eax,[edx+PrLdtDescriptor+4] mov [ecx+KGDT_LDT+4],eax ; Set up int 21 descriptor of IDT. If the process does not have Ldt, it ; should never make any int 21 call. If it does, an exception is generated. ; If the process has Ldt, we need to update int21 entry of LDT for the process. ; Note the Int21Descriptor of the process may simply indicate an invalid ; entry. In which case, the int 21 will be trapped to the kernel. mov ecx, PCR[PcIdt] mov eax, [edx+PrInt21Descriptor] mov [ecx+21h*8], eax mov eax, [edx+PrInt21Descriptor+4] mov [ecx+21h*8+4], eax mov eax,KGDT_LDT ;@@32-bit op to avoid prefix ; Load LDTR kisp10: lldt ax stdRET _KiSwapProcess if DBG kisp_error1: int 3 kisp_error: int 3 endif stdENDP _KiSwapProcess page , 132 subttl "Adjust TSS ESP0 value" ; VOID ; KiAdjustEsp0 ( ; IN PKTRAP_FRAME TrapFrame ; ) ; Routine Description: ; This routine puts the apropriate ESP0 value in the esp0 field of the ; TSS. This allows protect mode and V86 mode to use the same stack ; frame. The ESP0 value for protected mode is 16 bytes lower than ; for V86 mode to compensate for the missing segment registers. ; Arguments: ; TrapFrame - Supplies a pointer to the TrapFrame. ; Return Value: ; None. cPublicProc _Ki386AdjustEsp0 ,1 stdCall _KeGetCurrentThread mov edx,[esp + 4] ; edx -> trap frame mov eax,[eax]+thInitialStack ; eax = base of stack test dword ptr [edx]+TsEFlags,EFLAGS_V86_MASK ; is this a V86 frame? jnz short ae10 sub eax,TsV86Gs - TsHardwareSegSS ; compensate for missing regs ae10: sub eax,NPX_FRAME_LENGTH pushfd ; Make sure we don't move cli ; processors while we do this mov edx,PCR[PcTss] mov [edx]+TssEsp0,eax ; set Esp0 value popfd stdRET _Ki386AdjustEsp0 stdENDP _Ki386AdjustEsp0 _TEXT$00 ends end