630 lines
15 KiB
C
630 lines
15 KiB
C
/*++
|
||
|
||
Copyright (c) 1989 Microsoft Corporation
|
||
|
||
Module Name:
|
||
|
||
zeropage.c
|
||
|
||
Abstract:
|
||
|
||
This module contains the zero page thread for memory management.
|
||
|
||
Author:
|
||
|
||
Lou Perazzoli (loup) 6-Apr-1991
|
||
Landy Wang (landyw) 02-June-1997
|
||
|
||
Revision History:
|
||
|
||
--*/
|
||
|
||
#include "mi.h"
|
||
|
||
#define MM_ZERO_PAGE_OBJECT 0
|
||
#define PO_SYS_IDLE_OBJECT 1
|
||
#define NUMBER_WAIT_OBJECTS 2
|
||
|
||
#define MACHINE_ZERO_PAGE(ZeroBase,NumberOfBytes) KeZeroPagesFromIdleThread(ZeroBase,NumberOfBytes)
|
||
|
||
LOGICAL MiZeroingDisabled = FALSE;
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
LONG MiNextZeroProcessor = (LONG)-1;
|
||
|
||
#ifdef ALLOC_PRAGMA
|
||
#pragma alloc_text(INIT,MiStartZeroPageWorkers)
|
||
#endif
|
||
|
||
#endif
|
||
|
||
VOID
|
||
MmZeroPageThread (
|
||
VOID
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
Implements the NT zeroing page thread. This thread runs
|
||
at priority zero and removes a page from the free list,
|
||
zeroes it, and places it on the zeroed page list.
|
||
|
||
Arguments:
|
||
|
||
None.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
Environment:
|
||
|
||
Kernel mode.
|
||
|
||
--*/
|
||
|
||
{
|
||
KIRQL OldIrql;
|
||
PFN_NUMBER PageFrame1;
|
||
PFN_NUMBER PageFrame;
|
||
PMMPFN Pfn1;
|
||
PKTHREAD Thread;
|
||
PVOID ZeroBase;
|
||
PVOID WaitObjects[NUMBER_WAIT_OBJECTS];
|
||
NTSTATUS Status;
|
||
PVOID StartVa;
|
||
PVOID EndVa;
|
||
PFN_COUNT PagesToZero;
|
||
PFN_COUNT MaximumPagesToZero;
|
||
ULONG Color;
|
||
ULONG StartColor;
|
||
PMMPFN PfnAllocation;
|
||
|
||
#if defined(MI_MULTINODE)
|
||
|
||
ULONG i;
|
||
ULONG n;
|
||
ULONG LastNodeZeroing;
|
||
|
||
n = 0;
|
||
LastNodeZeroing = 0;
|
||
#endif
|
||
|
||
//
|
||
// Before this becomes the zero page thread, free the kernel
|
||
// initialization code.
|
||
//
|
||
|
||
MiFindInitializationCode (&StartVa, &EndVa);
|
||
|
||
if (StartVa != NULL) {
|
||
MiFreeInitializationCode (StartVa, EndVa);
|
||
}
|
||
|
||
MaximumPagesToZero = 1;
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
//
|
||
// Zero groups of pages at once to reduce PFN lock contention.
|
||
// Charge commitment as well as resident available up front since
|
||
// zeroing may get starved priority-wise.
|
||
//
|
||
// Note using MmSecondaryColors here would be excessively wasteful
|
||
// on NUMA systems. MmSecondaryColorMask + 1 is correct for all platforms.
|
||
//
|
||
|
||
PagesToZero = MmSecondaryColorMask + 1;
|
||
|
||
if (PagesToZero > NUMBER_OF_ZEROING_PTES) {
|
||
PagesToZero = NUMBER_OF_ZEROING_PTES;
|
||
}
|
||
|
||
if (MiChargeCommitment (PagesToZero, NULL) == TRUE) {
|
||
|
||
LOCK_PFN (OldIrql);
|
||
|
||
//
|
||
// Check to make sure the physical pages are available.
|
||
//
|
||
|
||
if (MI_NONPAGABLE_MEMORY_AVAILABLE() > (SPFN_NUMBER)(PagesToZero)) {
|
||
MI_DECREMENT_RESIDENT_AVAILABLE (PagesToZero,
|
||
MM_RESAVAIL_ALLOCATE_ZERO_PAGE_CLUSTERS);
|
||
MaximumPagesToZero = PagesToZero;
|
||
}
|
||
|
||
UNLOCK_PFN (OldIrql);
|
||
}
|
||
|
||
#endif
|
||
|
||
//
|
||
// The following code sets the current thread's base priority to zero
|
||
// and then sets its current priority to zero. This ensures that the
|
||
// thread always runs at a priority of zero.
|
||
//
|
||
|
||
Thread = KeGetCurrentThread ();
|
||
Thread->BasePriority = 0;
|
||
KeSetPriorityThread (Thread, 0);
|
||
|
||
//
|
||
// Initialize wait object array for multiple wait
|
||
//
|
||
|
||
WaitObjects[MM_ZERO_PAGE_OBJECT] = &MmZeroingPageEvent;
|
||
WaitObjects[PO_SYS_IDLE_OBJECT] = &PoSystemIdleTimer;
|
||
|
||
Color = 0;
|
||
PfnAllocation = (PMMPFN) MM_EMPTY_LIST;
|
||
|
||
//
|
||
// Loop forever zeroing pages.
|
||
//
|
||
|
||
do {
|
||
|
||
//
|
||
// Wait until there are at least MmZeroPageMinimum pages
|
||
// on the free list.
|
||
//
|
||
|
||
Status = KeWaitForMultipleObjects (NUMBER_WAIT_OBJECTS,
|
||
WaitObjects,
|
||
WaitAny,
|
||
WrFreePage,
|
||
KernelMode,
|
||
FALSE,
|
||
NULL,
|
||
NULL);
|
||
|
||
if (Status == PO_SYS_IDLE_OBJECT) {
|
||
PoSystemIdleWorker (TRUE);
|
||
continue;
|
||
}
|
||
|
||
PagesToZero = 0;
|
||
|
||
LOCK_PFN (OldIrql);
|
||
|
||
do {
|
||
|
||
if (MmFreePageListHead.Total == 0) {
|
||
|
||
//
|
||
// No pages on the free list at this time, wait for
|
||
// some more.
|
||
//
|
||
|
||
MmZeroingPageThreadActive = FALSE;
|
||
UNLOCK_PFN (OldIrql);
|
||
break;
|
||
}
|
||
|
||
if (MiZeroingDisabled == TRUE) {
|
||
MmZeroingPageThreadActive = FALSE;
|
||
UNLOCK_PFN (OldIrql);
|
||
KeDelayExecutionThread (KernelMode,
|
||
FALSE,
|
||
(PLARGE_INTEGER)&MmHalfSecond);
|
||
break;
|
||
}
|
||
|
||
#if defined(MI_MULTINODE)
|
||
|
||
//
|
||
// In a multinode system, zero pages by node. Resume on
|
||
// the last node examined, find a node with free pages that
|
||
// need to be zeroed.
|
||
//
|
||
|
||
if (KeNumberNodes > 1) {
|
||
|
||
n = LastNodeZeroing;
|
||
|
||
for (i = 0; i < KeNumberNodes; i += 1) {
|
||
if (KeNodeBlock[n]->FreeCount[FreePageList] != 0) {
|
||
break;
|
||
}
|
||
n = (n + 1) % KeNumberNodes;
|
||
}
|
||
|
||
ASSERT (i != KeNumberNodes);
|
||
ASSERT (KeNodeBlock[n]->FreeCount[FreePageList] != 0);
|
||
|
||
if (n != LastNodeZeroing) {
|
||
Color = KeNodeBlock[n]->MmShiftedColor;
|
||
}
|
||
}
|
||
#endif
|
||
|
||
ASSERT (PagesToZero == 0);
|
||
|
||
StartColor = Color;
|
||
|
||
do {
|
||
|
||
PageFrame = MmFreePagesByColor[FreePageList][Color].Flink;
|
||
|
||
if (PageFrame != MM_EMPTY_LIST) {
|
||
|
||
PageFrame1 = MiRemoveAnyPage (Color);
|
||
|
||
ASSERT (PageFrame == PageFrame1);
|
||
|
||
Pfn1 = MI_PFN_ELEMENT (PageFrame);
|
||
|
||
Pfn1->u1.Flink = (PFN_NUMBER) PfnAllocation;
|
||
|
||
//
|
||
// Temporarily mark the page as bad so that contiguous
|
||
// memory allocators won't steal it when we release
|
||
// the PFN lock below. This also prevents the
|
||
// MiIdentifyPfn code from trying to identify it as
|
||
// we haven't filled in all the fields yet.
|
||
//
|
||
|
||
Pfn1->u3.e1.PageLocation = BadPageList;
|
||
|
||
PfnAllocation = Pfn1;
|
||
|
||
PagesToZero += 1;
|
||
}
|
||
|
||
//
|
||
// March to the next color - this will be used to finish
|
||
// filling the current chunk or to start the next one.
|
||
//
|
||
|
||
Color = (Color & ~MmSecondaryColorMask) |
|
||
((Color + 1) & MmSecondaryColorMask);
|
||
|
||
if (PagesToZero == MaximumPagesToZero) {
|
||
break;
|
||
}
|
||
|
||
if (Color == StartColor) {
|
||
break;
|
||
}
|
||
|
||
} while (TRUE);
|
||
|
||
ASSERT (PfnAllocation != (PMMPFN) MM_EMPTY_LIST);
|
||
|
||
UNLOCK_PFN (OldIrql);
|
||
|
||
ZeroBase = MiMapPagesToZeroInHyperSpace (PfnAllocation, PagesToZero);
|
||
|
||
#if defined(MI_MULTINODE)
|
||
|
||
//
|
||
// If a node switch is in order, do it now that the PFN
|
||
// lock has been released.
|
||
//
|
||
|
||
if ((KeNumberNodes > 1) && (n != LastNodeZeroing)) {
|
||
LastNodeZeroing = n;
|
||
KeFindFirstSetLeftAffinity (KeNodeBlock[n]->ProcessorMask, &i);
|
||
KeSetIdealProcessorThread (Thread, (UCHAR)i);
|
||
}
|
||
|
||
#endif
|
||
|
||
MACHINE_ZERO_PAGE (ZeroBase, PagesToZero << PAGE_SHIFT);
|
||
|
||
#if 0
|
||
ASSERT (RtlCompareMemoryUlong (ZeroBase, PagesToZero << PAGE_SHIFT, 0) == PagesToZero << PAGE_SHIFT);
|
||
#endif
|
||
|
||
MiUnmapPagesInZeroSpace (ZeroBase, PagesToZero);
|
||
|
||
PagesToZero = 0;
|
||
|
||
Pfn1 = PfnAllocation;
|
||
|
||
LOCK_PFN (OldIrql);
|
||
|
||
do {
|
||
|
||
PageFrame = MI_PFN_ELEMENT_TO_INDEX (Pfn1);
|
||
|
||
Pfn1 = (PMMPFN) Pfn1->u1.Flink;
|
||
|
||
MiInsertPageInList (&MmZeroedPageListHead, PageFrame);
|
||
|
||
} while (Pfn1 != (PMMPFN) MM_EMPTY_LIST);
|
||
|
||
//
|
||
// We just finished processing a cluster of pages - briefly
|
||
// release the PFN lock to allow other threads to make progress.
|
||
//
|
||
|
||
UNLOCK_PFN (OldIrql);
|
||
|
||
PfnAllocation = (PMMPFN) MM_EMPTY_LIST;
|
||
|
||
LOCK_PFN (OldIrql);
|
||
|
||
} while (TRUE);
|
||
|
||
} while (TRUE);
|
||
}
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
|
||
VOID
|
||
MiZeroPageWorker (
|
||
IN PVOID Context
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This routine is the worker routine executed by all processors so that
|
||
initial page zeroing occurs in parallel.
|
||
|
||
Arguments:
|
||
|
||
Context - Supplies a pointer to the workitem.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
Environment:
|
||
|
||
Kernel mode initialization time, PASSIVE_LEVEL. Because this is INIT
|
||
only code, don't bother charging commit for the pages.
|
||
|
||
--*/
|
||
|
||
{
|
||
MMPTE TempPte;
|
||
PMMPTE PointerPte;
|
||
KAFFINITY Affinity;
|
||
KIRQL OldIrql;
|
||
PVOID ZeroBase;
|
||
PKTHREAD Thread;
|
||
CCHAR OldProcessor;
|
||
SCHAR OldBasePriority;
|
||
KPRIORITY OldPriority;
|
||
PWORK_QUEUE_ITEM WorkItem;
|
||
PMMPFN Pfn1;
|
||
PFN_NUMBER NewPage;
|
||
PFN_NUMBER PageFrame;
|
||
#if defined(MI_MULTINODE)
|
||
PKNODE Node;
|
||
ULONG Color;
|
||
ULONG FinalColor;
|
||
#endif
|
||
|
||
WorkItem = (PWORK_QUEUE_ITEM) Context;
|
||
|
||
ExFreePool (WorkItem);
|
||
|
||
TempPte = ValidKernelPte;
|
||
|
||
//
|
||
// The following code sets the current thread's base and current priorities
|
||
// to one so all other code (except the zero page thread) can preempt it.
|
||
//
|
||
|
||
Thread = KeGetCurrentThread ();
|
||
OldBasePriority = Thread->BasePriority;
|
||
Thread->BasePriority = 1;
|
||
OldPriority = KeSetPriorityThread (Thread, 1);
|
||
|
||
//
|
||
// Dispatch each worker thread to the next processor in line.
|
||
//
|
||
|
||
OldProcessor = (CCHAR) InterlockedIncrement (&MiNextZeroProcessor);
|
||
|
||
Affinity = AFFINITY_MASK (OldProcessor);
|
||
Affinity = KeSetAffinityThread (Thread, Affinity);
|
||
|
||
//
|
||
// Zero all local pages.
|
||
//
|
||
|
||
#if defined(MI_MULTINODE)
|
||
if (KeNumberNodes > 1) {
|
||
Node = KeGetCurrentNode ();
|
||
Color = Node->MmShiftedColor;
|
||
FinalColor = Color + MmSecondaryColorMask + 1;
|
||
}
|
||
else {
|
||
SATISFY_OVERZEALOUS_COMPILER (Node = NULL);
|
||
SATISFY_OVERZEALOUS_COMPILER (Color = 0);
|
||
SATISFY_OVERZEALOUS_COMPILER (FinalColor = 0);
|
||
}
|
||
#endif
|
||
|
||
LOCK_PFN (OldIrql);
|
||
|
||
do {
|
||
|
||
#if defined(MI_MULTINODE)
|
||
|
||
//
|
||
// In a multinode system, zero pages by node.
|
||
//
|
||
|
||
if (KeNumberNodes > 1) {
|
||
|
||
if (Node->FreeCount[FreePageList] == 0) {
|
||
|
||
//
|
||
// No pages on the free list at this time, bail.
|
||
//
|
||
|
||
UNLOCK_PFN (OldIrql);
|
||
break;
|
||
}
|
||
|
||
//
|
||
// Must start with a color MiRemoveAnyPage will
|
||
// satisfy from the free list otherwise it will
|
||
// return an already zeroed page.
|
||
//
|
||
|
||
while (MmFreePagesByColor[FreePageList][Color].Flink == MM_EMPTY_LIST) {
|
||
//
|
||
// No pages on this free list color, march to the next one.
|
||
//
|
||
|
||
Color += 1;
|
||
if (Color == FinalColor) {
|
||
UNLOCK_PFN (OldIrql);
|
||
goto ZeroingFinished;
|
||
}
|
||
}
|
||
|
||
PageFrame = MiRemoveAnyPage (Color);
|
||
}
|
||
else {
|
||
#endif
|
||
if (MmFreePageListHead.Total == 0) {
|
||
|
||
//
|
||
// No pages on the free list at this time, bail.
|
||
//
|
||
|
||
UNLOCK_PFN (OldIrql);
|
||
break;
|
||
}
|
||
|
||
PageFrame = MmFreePageListHead.Flink;
|
||
ASSERT (PageFrame != MM_EMPTY_LIST);
|
||
|
||
Pfn1 = MI_PFN_ELEMENT(PageFrame);
|
||
|
||
NewPage = MiRemoveAnyPage (MI_GET_COLOR_FROM_LIST_ENTRY(PageFrame, Pfn1));
|
||
if (NewPage != PageFrame) {
|
||
|
||
//
|
||
// Someone has removed a page from the colored lists
|
||
// chain without updating the freelist chain.
|
||
//
|
||
|
||
KeBugCheckEx (PFN_LIST_CORRUPT,
|
||
0x8F,
|
||
NewPage,
|
||
PageFrame,
|
||
0);
|
||
}
|
||
#if defined(MI_MULTINODE)
|
||
}
|
||
#endif
|
||
|
||
//
|
||
// Use system PTEs instead of hyperspace to zero the page so that
|
||
// a spinlock (ie: interrupts blocked) is not held while zeroing.
|
||
// Since system PTE acquisition is lock free and the TB lazy flushed,
|
||
// this is perhaps the best path regardless.
|
||
//
|
||
|
||
UNLOCK_PFN (OldIrql);
|
||
|
||
PointerPte = MiReserveSystemPtes (1, SystemPteSpace);
|
||
|
||
if (PointerPte == NULL) {
|
||
|
||
//
|
||
// Put this page back on the freelist.
|
||
//
|
||
|
||
LOCK_PFN (OldIrql);
|
||
|
||
MiInsertPageInFreeList (PageFrame);
|
||
|
||
UNLOCK_PFN (OldIrql);
|
||
|
||
break;
|
||
}
|
||
|
||
ASSERT (PointerPte->u.Hard.Valid == 0);
|
||
|
||
ZeroBase = MiGetVirtualAddressMappedByPte (PointerPte);
|
||
|
||
TempPte.u.Hard.PageFrameNumber = PageFrame;
|
||
MI_WRITE_VALID_PTE (PointerPte, TempPte);
|
||
|
||
KeZeroPages (ZeroBase, PAGE_SIZE);
|
||
|
||
MiReleaseSystemPtes (PointerPte, 1, SystemPteSpace);
|
||
|
||
LOCK_PFN (OldIrql);
|
||
|
||
MiInsertPageInList (&MmZeroedPageListHead, PageFrame);
|
||
|
||
} while (TRUE);
|
||
|
||
#if defined(MI_MULTINODE)
|
||
ZeroingFinished:
|
||
#endif
|
||
|
||
//
|
||
// Restore the entry thread priority and processor affinity.
|
||
//
|
||
|
||
KeSetAffinityThread (Thread, Affinity);
|
||
|
||
KeSetPriorityThread (Thread, OldPriority);
|
||
Thread->BasePriority = OldBasePriority;
|
||
}
|
||
|
||
|
||
VOID
|
||
MiStartZeroPageWorkers (
|
||
VOID
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This routine starts the zero page worker threads.
|
||
|
||
Arguments:
|
||
|
||
None.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
Environment:
|
||
|
||
Kernel mode initialization phase 1, PASSIVE_LEVEL.
|
||
|
||
--*/
|
||
|
||
{
|
||
ULONG i;
|
||
PWORK_QUEUE_ITEM WorkItem;
|
||
|
||
for (i = 0; i < (ULONG) KeNumberProcessors; i += 1) {
|
||
|
||
WorkItem = ExAllocatePoolWithTag (NonPagedPool,
|
||
sizeof (WORK_QUEUE_ITEM),
|
||
'wZmM');
|
||
|
||
if (WorkItem == NULL) {
|
||
break;
|
||
}
|
||
|
||
ExInitializeWorkItem (WorkItem, MiZeroPageWorker, (PVOID) WorkItem);
|
||
|
||
ExQueueWorkItem (WorkItem, CriticalWorkQueue);
|
||
}
|
||
}
|
||
|
||
#endif
|