1750 lines
45 KiB
C
1750 lines
45 KiB
C
/*++
|
|
|
|
Copyright (c) 1989 Microsoft Corporation
|
|
|
|
Module Name:
|
|
|
|
pfnlist.c
|
|
|
|
Abstract:
|
|
|
|
This module contains the routines to manipulate pages within the
|
|
Page Frame Database.
|
|
|
|
Author:
|
|
|
|
Lou Perazzoli (loup) 4-Apr-1989
|
|
Landy Wang (landyw) 02-June-1997
|
|
|
|
Revision History:
|
|
|
|
--*/
|
|
#include "mi.h"
|
|
|
|
#define MM_LOW_LIMIT 2
|
|
#define MM_HIGH_LIMIT 19
|
|
|
|
KEVENT MmAvailablePagesEventHigh;
|
|
|
|
ULONG MmTransitionPrivatePages;
|
|
ULONG MmTransitionSharedPages;
|
|
|
|
#define MI_TALLY_TRANSITION_PAGE_ADDITION(Pfn) \
|
|
if (Pfn->u3.e1.PrototypePte) { \
|
|
MmTransitionSharedPages += 1; \
|
|
} \
|
|
else { \
|
|
MmTransitionPrivatePages += 1; \
|
|
} \
|
|
ASSERT (MmTransitionPrivatePages + MmTransitionSharedPages == MmStandbyPageListHead.Total + MmModifiedPageListHead.Total + MmModifiedNoWritePageListHead.Total);
|
|
|
|
#define MI_TALLY_TRANSITION_PAGE_REMOVAL(Pfn) \
|
|
if (Pfn->u3.e1.PrototypePte) { \
|
|
MmTransitionSharedPages -= 1; \
|
|
} \
|
|
else { \
|
|
MmTransitionPrivatePages -= 1; \
|
|
} \
|
|
ASSERT (MmTransitionPrivatePages + MmTransitionSharedPages == MmStandbyPageListHead.Total + MmModifiedPageListHead.Total + MmModifiedNoWritePageListHead.Total);
|
|
|
|
VOID
|
|
MiRemovePageByColor (
|
|
IN PFN_NUMBER Page,
|
|
IN ULONG PageColor
|
|
);
|
|
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiInsertPageInList (
|
|
IN PMMPFNLIST ListHead,
|
|
IN PFN_NUMBER PageFrameIndex
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This procedure inserts a page at the end of the specified list (free,
|
|
standby, bad, zeroed, modified).
|
|
|
|
|
|
Arguments:
|
|
|
|
ListHead - Supplies the list of the list in which to insert the
|
|
specified physical page.
|
|
|
|
PageFrameIndex - Supplies the physical page number to insert in the
|
|
list.
|
|
|
|
Return Value:
|
|
|
|
none.
|
|
|
|
Environment:
|
|
|
|
Must be holding the PFN database mutex with APCs disabled.
|
|
|
|
--*/
|
|
|
|
{
|
|
PFN_NUMBER last;
|
|
PMMPFN Pfn1;
|
|
PMMPFN Pfn2;
|
|
ULONG Color;
|
|
|
|
MM_PFN_LOCK_ASSERT();
|
|
ASSERT ((PageFrameIndex != 0) &&
|
|
(PageFrameIndex <= MmHighestPhysicalPage) &&
|
|
(PageFrameIndex >= MmLowestPhysicalPage));
|
|
|
|
|
|
// Check to ensure the reference count for the page is zero.
|
|
|
|
|
|
Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
|
|
|
|
ASSERT (Pfn1->u3.e1.LockCharged == 0);
|
|
|
|
PERFINFO_INSERTINLIST(PageFrameIndex, ListHead);
|
|
|
|
#if DBG
|
|
if (MmDebug & MM_DBG_PAGE_REF_COUNT) {
|
|
|
|
PMMPTE PointerPte;
|
|
KIRQL OldIrql = 99;
|
|
|
|
if ((ListHead->ListName == StandbyPageList) ||
|
|
(ListHead->ListName == ModifiedPageList)) {
|
|
|
|
if ((Pfn1->u3.e1.PrototypePte == 1) &&
|
|
(MmIsAddressValid (Pfn1->PteAddress))) {
|
|
PointerPte = Pfn1->PteAddress;
|
|
} else {
|
|
|
|
|
|
// The page containing the prototype PTE is not valid,
|
|
// map the page into hyperspace and reference it that way.
|
|
|
|
|
|
PointerPte = MiMapPageInHyperSpace (Pfn1->PteFrame, &OldIrql);
|
|
PointerPte = (PMMPTE)((PCHAR)PointerPte +
|
|
MiGetByteOffset(Pfn1->PteAddress));
|
|
}
|
|
|
|
ASSERT ((MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (PointerPte) == PageFrameIndex) ||
|
|
(MI_GET_PAGE_FRAME_FROM_PTE (PointerPte) == PageFrameIndex));
|
|
ASSERT (PointerPte->u.Soft.Transition == 1);
|
|
ASSERT (PointerPte->u.Soft.Prototype == 0);
|
|
if (OldIrql != 99) {
|
|
MiUnmapPageInHyperSpace (OldIrql)
|
|
}
|
|
}
|
|
}
|
|
#endif
|
|
|
|
#if PFN_CONSISTENCY
|
|
if (ListHead == &MmFreePageListHead) {
|
|
if (Pfn1->u2.ShareCount != 0) {
|
|
KeBugCheckEx (PFN_LIST_CORRUPT,
|
|
0x91,
|
|
PageFrameIndex,
|
|
Pfn1->u2.ShareCount,
|
|
Pfn1->u3.e2.ReferenceCount);
|
|
}
|
|
}
|
|
else if (ListHead == &MmZeroedPageListHead) {
|
|
if (Pfn1->u2.ShareCount != 0) {
|
|
KeBugCheckEx (PFN_LIST_CORRUPT,
|
|
0x92,
|
|
PageFrameIndex,
|
|
Pfn1->u2.ShareCount,
|
|
Pfn1->u3.e2.ReferenceCount);
|
|
}
|
|
}
|
|
#endif
|
|
|
|
#if DBG
|
|
if ((ListHead->ListName == StandbyPageList) ||
|
|
(ListHead->ListName == ModifiedPageList)) {
|
|
if ((Pfn1->OriginalPte.u.Soft.Prototype == 0) &&
|
|
(Pfn1->OriginalPte.u.Soft.Transition == 1)) {
|
|
KeBugCheckEx (MEMORY_MANAGEMENT, 0x8888, 0,0,0);
|
|
}
|
|
}
|
|
#endif
|
|
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
|
|
ListHead->Total += 1; // One more page on the list.
|
|
|
|
|
|
// On MIPS R4000 modified pages destined for the paging file are
|
|
// kept on separate lists which group pages of the same color
|
|
// together
|
|
|
|
|
|
if (ListHead == &MmModifiedPageListHead) {
|
|
|
|
#if PFN_CONSISTENCY
|
|
if (Pfn1->u2.ShareCount != 0) {
|
|
KeBugCheckEx (PFN_LIST_CORRUPT,
|
|
0x90,
|
|
PageFrameIndex,
|
|
Pfn1->u2.ShareCount,
|
|
Pfn1->u3.e2.ReferenceCount);
|
|
}
|
|
#endif
|
|
|
|
if (Pfn1->OriginalPte.u.Soft.Prototype == 0) {
|
|
|
|
|
|
// This page is destined for the paging file (not
|
|
// a mapped file). Change the list head to the
|
|
// appropriate colored list head.
|
|
|
|
|
|
ListHead = &MmModifiedPageListByColor [Pfn1->u3.e1.PageColor];
|
|
ListHead->Total += 1;
|
|
MmTotalPagesForPagingFile += 1;
|
|
}
|
|
else {
|
|
|
|
|
|
// This page is destined for a mapped file (not
|
|
// the paging file). If there are no other pages currently
|
|
// destined for the mapped file, start our timer so that we can
|
|
// ensure that these pages make it to disk even if we don't pile
|
|
// up enough of them to trigger the modified page writer or need
|
|
// the memory. If we don't do this here, then for this scenario,
|
|
// only an orderly system shutdown will write them out (days,
|
|
// weeks, months or years later) and any power out in between
|
|
// means we'll have lost the data.
|
|
|
|
|
|
if (ListHead->Total - MmTotalPagesForPagingFile == 1) {
|
|
|
|
|
|
// Start the DPC timer because we're the first on the list.
|
|
|
|
|
|
if (MiTimerPending == FALSE) {
|
|
MiTimerPending = TRUE;
|
|
|
|
(VOID) KeSetTimerEx( &MiModifiedPageWriterTimer, MiModifiedPageLife, 0, &MiModifiedPageWriterTimerDpc );
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else if ((Pfn1->u3.e1.RemovalRequested == 1) &&
|
|
(ListHead->ListName <= StandbyPageList)) {
|
|
|
|
ListHead->Total -= 1; // Undo previous increment
|
|
|
|
if (ListHead->ListName == StandbyPageList) {
|
|
Pfn1->u3.e1.PageLocation = StandbyPageList;
|
|
MiRestoreTransitionPte (PageFrameIndex);
|
|
}
|
|
|
|
ListHead = MmPageLocationList[BadPageList];
|
|
ListHead->Total += 1; // One more page on the list.
|
|
}
|
|
|
|
|
|
last = ListHead->Blink;
|
|
if (last == MM_EMPTY_LIST) {
|
|
|
|
|
|
// List is empty add the page to the ListHead.
|
|
|
|
|
|
ListHead->Flink = PageFrameIndex;
|
|
} else {
|
|
Pfn2 = MI_PFN_ELEMENT (last);
|
|
Pfn2->u1.Flink = PageFrameIndex;
|
|
}
|
|
|
|
ListHead->Blink = PageFrameIndex;
|
|
Pfn1->u1.Flink = MM_EMPTY_LIST;
|
|
Pfn1->u2.Blink = last;
|
|
Pfn1->u3.e1.PageLocation = ListHead->ListName;
|
|
|
|
|
|
// If the page was placed on the free, standby or zeroed list,
|
|
// update the count of usable pages in the system. If the count
|
|
// transitions from 0 to 1, the event associated with available
|
|
// pages should become true.
|
|
|
|
|
|
if (ListHead->ListName <= StandbyPageList) {
|
|
MmAvailablePages += 1;
|
|
|
|
|
|
// A page has just become available, check to see if the
|
|
// page wait events should be signalled.
|
|
|
|
|
|
if (MmAvailablePages == MM_LOW_LIMIT) {
|
|
KeSetEvent (&MmAvailablePagesEvent, 0, FALSE);
|
|
} else if (MmAvailablePages == MM_HIGH_LIMIT) {
|
|
KeSetEvent (&MmAvailablePagesEventHigh, 0, FALSE);
|
|
}
|
|
|
|
if (ListHead->ListName <= FreePageList) {
|
|
|
|
ASSERT (Pfn1->u3.e1.InPageError == 0);
|
|
|
|
|
|
// We are adding a page to the free or zeroed page list.
|
|
// Add the page to the end of the correct colored page list.
|
|
|
|
|
|
Color = MI_GET_SECONDARY_COLOR (PageFrameIndex, Pfn1);
|
|
ASSERT (Pfn1->u3.e1.PageColor == MI_GET_COLOR_FROM_SECONDARY(Color));
|
|
|
|
if (MmFreePagesByColor[ListHead->ListName][Color].Flink ==
|
|
MM_EMPTY_LIST) {
|
|
|
|
|
|
// This list is empty, add this as the first and last
|
|
// entry.
|
|
|
|
|
|
MmFreePagesByColor[ListHead->ListName][Color].Flink =
|
|
PageFrameIndex;
|
|
MmFreePagesByColor[ListHead->ListName][Color].Blink =
|
|
(PVOID)Pfn1;
|
|
} else {
|
|
Pfn2 = (PMMPFN)MmFreePagesByColor[ListHead->ListName][Color].Blink;
|
|
Pfn2->OriginalPte.u.Long = PageFrameIndex;
|
|
MmFreePagesByColor[ListHead->ListName][Color].Blink = (PVOID)Pfn1;
|
|
}
|
|
Pfn1->OriginalPte.u.Long = MM_EMPTY_LIST;
|
|
|
|
if (ListHead->ListName == ZeroedPageList) {
|
|
MI_BARRIER_STAMP_ZEROED_PAGE (&Pfn1->PteFrame);
|
|
}
|
|
}
|
|
else {
|
|
|
|
|
|
// Transition page list so tally it appropriately.
|
|
|
|
|
|
MI_TALLY_TRANSITION_PAGE_ADDITION (Pfn1);
|
|
}
|
|
|
|
if ((ListHead->ListName == FreePageList) &&
|
|
(MmFreePageListHead.Total >= MmMinimumFreePagesToZero) &&
|
|
(MmZeroingPageThreadActive == FALSE)) {
|
|
|
|
|
|
// There are enough pages on the free list, start
|
|
// the zeroing page thread.
|
|
|
|
|
|
MmZeroingPageThreadActive = TRUE;
|
|
KeSetEvent (&MmZeroingPageEvent, 0, FALSE);
|
|
}
|
|
return;
|
|
}
|
|
|
|
|
|
// Check to see if there are too many modified pages.
|
|
|
|
|
|
if (ListHead->ListName == ModifiedPageList) {
|
|
|
|
|
|
// Transition page list so tally it appropriately.
|
|
|
|
|
|
MI_TALLY_TRANSITION_PAGE_ADDITION (Pfn1);
|
|
|
|
if (Pfn1->OriginalPte.u.Soft.Prototype == 0) {
|
|
ASSERT (Pfn1->OriginalPte.u.Soft.PageFileHigh == 0);
|
|
}
|
|
|
|
PsGetCurrentProcess()->ModifiedPageCount += 1;
|
|
if (MmModifiedPageListHead.Total >= MmModifiedPageMaximum ) {
|
|
|
|
|
|
// Start the modified page writer.
|
|
|
|
|
|
KeSetEvent (&MmModifiedPageWriterEvent, 0, FALSE);
|
|
}
|
|
}
|
|
else if (ListHead->ListName == ModifiedNoWritePageList) {
|
|
MI_TALLY_TRANSITION_PAGE_ADDITION (Pfn1);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiInsertStandbyListAtFront (
|
|
IN PFN_NUMBER PageFrameIndex
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This procedure inserts a page at the front of the standby list.
|
|
|
|
Arguments:
|
|
|
|
PageFrameIndex - Supplies the physical page number to insert in the
|
|
list.
|
|
|
|
Return Value:
|
|
|
|
none.
|
|
|
|
Environment:
|
|
|
|
Must be holding the PFN database mutex with APCs disabled.
|
|
|
|
--*/
|
|
|
|
{
|
|
PFN_NUMBER first;
|
|
IN PMMPFNLIST ListHead;
|
|
PMMPFN Pfn1;
|
|
PMMPFN Pfn2;
|
|
|
|
MM_PFN_LOCK_ASSERT();
|
|
ASSERT ((PageFrameIndex != 0) && (PageFrameIndex <= MmHighestPhysicalPage) &&
|
|
(PageFrameIndex >= MmLowestPhysicalPage));
|
|
|
|
|
|
// Check to ensure the reference count for the page
|
|
// is zero.
|
|
|
|
|
|
Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
|
|
|
|
PERFINFO_INSERT_FRONT_STANDBY(PageFrameIndex);
|
|
|
|
#if DBG
|
|
if (MmDebug & MM_DBG_PAGE_REF_COUNT) {
|
|
|
|
PMMPTE PointerPte;
|
|
KIRQL OldIrql = 99;
|
|
|
|
if ((Pfn1->u3.e1.PrototypePte == 1) &&
|
|
(MmIsAddressValid (Pfn1->PteAddress))) {
|
|
PointerPte = Pfn1->PteAddress;
|
|
} else {
|
|
|
|
|
|
// The page containing the prototype PTE is not valid,
|
|
// map the page into hyperspace and reference it that way.
|
|
|
|
|
|
PointerPte = MiMapPageInHyperSpace (Pfn1->PteFrame, &OldIrql);
|
|
PointerPte = (PMMPTE)((PCHAR)PointerPte +
|
|
MiGetByteOffset(Pfn1->PteAddress));
|
|
}
|
|
|
|
ASSERT ((MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (PointerPte) == PageFrameIndex) ||
|
|
(MI_GET_PAGE_FRAME_FROM_PTE (PointerPte) == PageFrameIndex));
|
|
ASSERT (PointerPte->u.Soft.Transition == 1);
|
|
ASSERT (PointerPte->u.Soft.Prototype == 0);
|
|
if (OldIrql != 99) {
|
|
MiUnmapPageInHyperSpace (OldIrql)
|
|
}
|
|
}
|
|
|
|
if ((Pfn1->OriginalPte.u.Soft.Prototype == 0) &&
|
|
(Pfn1->OriginalPte.u.Soft.Transition == 1)) {
|
|
KeBugCheckEx (MEMORY_MANAGEMENT, 0x8889, 0,0,0);
|
|
}
|
|
#endif
|
|
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
|
|
MmTransitionSharedPages += 1;
|
|
|
|
MmStandbyPageListHead.Total += 1; // One more page on the list.
|
|
|
|
ASSERT (MmTransitionPrivatePages + MmTransitionSharedPages == MmStandbyPageListHead.Total + MmModifiedPageListHead.Total + MmModifiedNoWritePageListHead.Total);
|
|
|
|
ListHead = &MmStandbyPageListHead;
|
|
|
|
first = ListHead->Flink;
|
|
if (first == MM_EMPTY_LIST) {
|
|
|
|
|
|
// List is empty add the page to the ListHead.
|
|
|
|
|
|
ListHead->Blink = PageFrameIndex;
|
|
} else {
|
|
Pfn2 = MI_PFN_ELEMENT (first);
|
|
Pfn2->u2.Blink = PageFrameIndex;
|
|
}
|
|
|
|
ListHead->Flink = PageFrameIndex;
|
|
Pfn1->u2.Blink = MM_EMPTY_LIST;
|
|
Pfn1->u1.Flink = first;
|
|
Pfn1->u3.e1.PageLocation = StandbyPageList;
|
|
|
|
|
|
// If the page was placed on the free, standby or zeroed list,
|
|
// update the count of usable pages in the system. If the count
|
|
// transitions from 0 to 1, the event associated with available
|
|
// pages should become true.
|
|
|
|
|
|
MmAvailablePages += 1;
|
|
|
|
|
|
// A page has just become available, check to see if the
|
|
// page wait events should be signalled.
|
|
|
|
|
|
if (MmAvailablePages == MM_LOW_LIMIT) {
|
|
KeSetEvent (&MmAvailablePagesEvent, 0, FALSE);
|
|
} else if (MmAvailablePages == MM_HIGH_LIMIT) {
|
|
KeSetEvent (&MmAvailablePagesEventHigh, 0, FALSE);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
PFN_NUMBER //PageFrameIndex
|
|
FASTCALL
|
|
MiRemovePageFromList (
|
|
IN PMMPFNLIST ListHead
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This procedure removes a page from the head of the specified list (free,
|
|
standby, zeroed, modified).
|
|
|
|
This routine clears the flags word in the PFN database, hence the
|
|
PFN information for this page must be initialized.
|
|
|
|
Arguments:
|
|
|
|
ListHead - Supplies the list of the list in which to remove the
|
|
specified physical page.
|
|
|
|
Return Value:
|
|
|
|
The physical page number removed from the specified list.
|
|
|
|
Environment:
|
|
|
|
Must be holding the PFN database mutex with APCs disabled.
|
|
|
|
--*/
|
|
|
|
{
|
|
PFN_NUMBER PageFrameIndex;
|
|
PMMPFN Pfn1;
|
|
PMMPFN Pfn2;
|
|
ULONG Color;
|
|
|
|
MM_PFN_LOCK_ASSERT();
|
|
|
|
|
|
// If the specified list is empty return MM_EMPTY_LIST.
|
|
|
|
|
|
if (ListHead->Total == 0) {
|
|
|
|
KdPrint(("MM:Attempting to remove page from empty list\n"));
|
|
KeBugCheckEx (PFN_LIST_CORRUPT, 1, (ULONG_PTR)ListHead, MmAvailablePages, 0);
|
|
return 0;
|
|
}
|
|
|
|
ASSERT (ListHead->ListName != ModifiedPageList);
|
|
|
|
|
|
// Decrement the count of pages on the list and remove the first
|
|
// page from the list.
|
|
|
|
|
|
ListHead->Total -= 1;
|
|
PageFrameIndex = ListHead->Flink;
|
|
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
|
|
|
|
PERFINFO_REMOVEPAGE(PageFrameIndex, PERFINFO_LOG_TYPE_REMOVEPAGEFROMLIST);
|
|
|
|
ListHead->Flink = Pfn1->u1.Flink;
|
|
|
|
|
|
// Zero the flink and blink in the pfn database element.
|
|
|
|
|
|
Pfn1->u1.Flink = 0; // Assumes Flink width is >= WsIndex width
|
|
Pfn1->u2.Blink = 0;
|
|
|
|
|
|
// If the last page was removed (the ListHead->Flink is now
|
|
// MM_EMPTY_LIST) make the listhead->Blink MM_EMPTY_LIST as well.
|
|
|
|
|
|
if (ListHead->Flink == MM_EMPTY_LIST) {
|
|
ListHead->Blink = MM_EMPTY_LIST;
|
|
} else {
|
|
|
|
|
|
// Make the PFN element point to MM_EMPTY_LIST signifying this
|
|
// is the last page in the list.
|
|
|
|
|
|
Pfn2 = MI_PFN_ELEMENT (ListHead->Flink);
|
|
Pfn2->u2.Blink = MM_EMPTY_LIST;
|
|
}
|
|
|
|
|
|
// Check to see if we now have one less page available.
|
|
|
|
|
|
if (ListHead->ListName <= StandbyPageList) {
|
|
MmAvailablePages -= 1;
|
|
|
|
if (ListHead->ListName == StandbyPageList) {
|
|
|
|
|
|
// This page is currently in transition, restore the PTE to
|
|
// its original contents so this page can be reused.
|
|
|
|
|
|
MI_TALLY_TRANSITION_PAGE_REMOVAL (Pfn1);
|
|
MiRestoreTransitionPte (PageFrameIndex);
|
|
}
|
|
|
|
if (MmAvailablePages < MmMinimumFreePages) {
|
|
|
|
|
|
// Obtain free pages.
|
|
|
|
|
|
MiObtainFreePages();
|
|
}
|
|
}
|
|
|
|
ASSERT ((PageFrameIndex != 0) &&
|
|
(PageFrameIndex <= MmHighestPhysicalPage) &&
|
|
(PageFrameIndex >= MmLowestPhysicalPage));
|
|
|
|
|
|
// Zero the PFN flags longword.
|
|
|
|
|
|
Color = Pfn1->u3.e1.PageColor;
|
|
ASSERT (Pfn1->u3.e1.RemovalRequested == 0);
|
|
Pfn1->u3.e2.ShortFlags = 0;
|
|
Pfn1->u3.e1.PageColor = Color;
|
|
Color = MI_GET_SECONDARY_COLOR (PageFrameIndex, Pfn1);
|
|
|
|
if (ListHead->ListName <= FreePageList) {
|
|
|
|
|
|
// Update the color lists.
|
|
|
|
|
|
ASSERT (MmFreePagesByColor[ListHead->ListName][Color].Flink == PageFrameIndex);
|
|
MmFreePagesByColor[ListHead->ListName][Color].Flink =
|
|
(PFN_NUMBER) Pfn1->OriginalPte.u.Long;
|
|
}
|
|
|
|
return PageFrameIndex;
|
|
}
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiUnlinkPageFromList (
|
|
IN PMMPFN Pfn
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This procedure removes a page from the middle of a list. This is
|
|
designed for the faulting of transition pages from the standby and
|
|
modified list and making them active and valid again.
|
|
|
|
Arguments:
|
|
|
|
Pfn - Supplies a pointer to the PFN database element for the physical
|
|
page to remove from the list.
|
|
|
|
Return Value:
|
|
|
|
none.
|
|
|
|
Environment:
|
|
|
|
Must be holding the PFN database mutex with APCs disabled.
|
|
|
|
--*/
|
|
|
|
{
|
|
PMMPFNLIST ListHead;
|
|
PFN_NUMBER Previous;
|
|
PFN_NUMBER Next;
|
|
PMMPFN Pfn2;
|
|
|
|
MM_PFN_LOCK_ASSERT();
|
|
|
|
PERFINFO_UNLINKPAGE((ULONG_PTR)(Pfn - MmPfnDatabase), Pfn->u3.e1.PageLocation);
|
|
|
|
|
|
// Page not on standby or modified list, check to see if the
|
|
// page is currently being written by the modified page
|
|
// writer, if so, just return this page. The reference
|
|
// count for the page will be incremented, so when the modified
|
|
// page write completes, the page will not be put back on
|
|
// the list, rather, it will remain active and valid.
|
|
|
|
|
|
if (Pfn->u3.e2.ReferenceCount > 0) {
|
|
|
|
|
|
// The page was not on any "transition lists", check to see
|
|
// if this has I/O in progress.
|
|
|
|
|
|
if (Pfn->u2.ShareCount == 0) {
|
|
#if DBG
|
|
if (MmDebug & MM_DBG_PAGE_IN_LIST) {
|
|
DbgPrint("unlinking page not in list...\n");
|
|
MiFormatPfn(Pfn);
|
|
}
|
|
#endif
|
|
return;
|
|
}
|
|
KdPrint(("MM:attempt to remove page from wrong page list\n"));
|
|
KeBugCheckEx (PFN_LIST_CORRUPT,
|
|
2,
|
|
Pfn - MmPfnDatabase,
|
|
MmHighestPhysicalPage,
|
|
Pfn->u3.e2.ReferenceCount);
|
|
return;
|
|
}
|
|
|
|
ListHead = MmPageLocationList[Pfn->u3.e1.PageLocation];
|
|
|
|
|
|
// Must not remove pages from free or zeroed without updating
|
|
// the colored lists.
|
|
|
|
|
|
ASSERT (ListHead->ListName >= StandbyPageList);
|
|
|
|
|
|
// On MIPS R4000 modified pages destined for the paging file are
|
|
// kept on separate lists which group pages of the same color
|
|
// together
|
|
|
|
|
|
if ((ListHead == &MmModifiedPageListHead) &&
|
|
(Pfn->OriginalPte.u.Soft.Prototype == 0)) {
|
|
|
|
|
|
// This page is destined for the paging file (not
|
|
// a mapped file). Change the list head to the
|
|
// appropriate colored list head.
|
|
|
|
|
|
ListHead->Total -= 1;
|
|
MmTotalPagesForPagingFile -= 1;
|
|
ListHead = &MmModifiedPageListByColor [Pfn->u3.e1.PageColor];
|
|
}
|
|
|
|
ASSERT (Pfn->u3.e1.WriteInProgress == 0);
|
|
ASSERT (Pfn->u3.e1.ReadInProgress == 0);
|
|
ASSERT (ListHead->Total != 0);
|
|
|
|
Next = Pfn->u1.Flink;
|
|
Pfn->u1.Flink = 0; // Assumes Flink width is >= WsIndex width
|
|
Previous = Pfn->u2.Blink;
|
|
Pfn->u2.Blink = 0;
|
|
|
|
if (Next == MM_EMPTY_LIST) {
|
|
ListHead->Blink = Previous;
|
|
} else {
|
|
Pfn2 = MI_PFN_ELEMENT(Next);
|
|
Pfn2->u2.Blink = Previous;
|
|
}
|
|
|
|
if (Previous == MM_EMPTY_LIST) {
|
|
ListHead->Flink = Next;
|
|
} else {
|
|
Pfn2 = MI_PFN_ELEMENT(Previous);
|
|
Pfn2->u1.Flink = Next;
|
|
}
|
|
|
|
ListHead->Total -= 1;
|
|
|
|
|
|
// Check to see if we now have one less page available.
|
|
|
|
|
|
if (ListHead->ListName <= StandbyPageList) {
|
|
MmAvailablePages -= 1;
|
|
|
|
if (ListHead->ListName == StandbyPageList) {
|
|
MI_TALLY_TRANSITION_PAGE_REMOVAL (Pfn);
|
|
}
|
|
|
|
if (MmAvailablePages < MmMinimumFreePages) {
|
|
|
|
|
|
// Obtain free pages.
|
|
|
|
|
|
MiObtainFreePages();
|
|
|
|
}
|
|
}
|
|
else if (ListHead->ListName == ModifiedPageList || ListHead->ListName == ModifiedNoWritePageList) {
|
|
MI_TALLY_TRANSITION_PAGE_REMOVAL (Pfn);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
VOID
|
|
MiUnlinkFreeOrZeroedPage (
|
|
IN PFN_NUMBER Page
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This procedure removes a page from the middle of a list. This is
|
|
designed for the removing of free or zeroed pages from the middle of
|
|
their lists.
|
|
|
|
Arguments:
|
|
|
|
Pfn - Supplies a pointer to the PFN database element for the physical
|
|
page to remove from the list.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
Environment:
|
|
|
|
Must be holding the PFN database mutex with APCs disabled.
|
|
|
|
--*/
|
|
|
|
{
|
|
PMMPFNLIST ListHead;
|
|
PFN_NUMBER Previous;
|
|
PFN_NUMBER Next;
|
|
PMMPFN Pfn2;
|
|
PMMPFN Pfn;
|
|
ULONG Color;
|
|
|
|
Pfn = MI_PFN_ELEMENT (Page);
|
|
|
|
MM_PFN_LOCK_ASSERT();
|
|
|
|
ListHead = MmPageLocationList[Pfn->u3.e1.PageLocation];
|
|
ASSERT (ListHead->Total != 0);
|
|
ListHead->Total -= 1;
|
|
|
|
ASSERT (ListHead->ListName <= FreePageList);
|
|
ASSERT (Pfn->u3.e1.WriteInProgress == 0);
|
|
ASSERT (Pfn->u3.e1.ReadInProgress == 0);
|
|
|
|
PERFINFO_UNLINKFREEPAGE((ULONG_PTR)(Pfn - MmPfnDatabase), Pfn->u3.e1.PageLocation);
|
|
|
|
Next = Pfn->u1.Flink;
|
|
Pfn->u1.Flink = 0; // Assumes Flink width is >= WsIndex width
|
|
Previous = Pfn->u2.Blink;
|
|
Pfn->u2.Blink = 0;
|
|
|
|
if (Next == MM_EMPTY_LIST) {
|
|
ListHead->Blink = Previous;
|
|
} else {
|
|
Pfn2 = MI_PFN_ELEMENT(Next);
|
|
Pfn2->u2.Blink = Previous;
|
|
}
|
|
|
|
if (Previous == MM_EMPTY_LIST) {
|
|
ListHead->Flink = Next;
|
|
} else {
|
|
Pfn2 = MI_PFN_ELEMENT(Previous);
|
|
Pfn2->u1.Flink = Next;
|
|
}
|
|
|
|
|
|
// We are removing a page from the middle of the free or zeroed page list.
|
|
// The secondary color tables must be updated at this time.
|
|
|
|
|
|
Color = MI_GET_SECONDARY_COLOR (Page, Pfn);
|
|
ASSERT (Pfn->u3.e1.PageColor == MI_GET_COLOR_FROM_SECONDARY(Color));
|
|
|
|
|
|
// Walk down the list and remove the page.
|
|
|
|
|
|
Next = MmFreePagesByColor[ListHead->ListName][Color].Flink;
|
|
if (Next == Page) {
|
|
MmFreePagesByColor[ListHead->ListName][Color].Flink =
|
|
(PFN_NUMBER) Pfn->OriginalPte.u.Long;
|
|
} else {
|
|
|
|
|
|
// Walk the list to find the parent.
|
|
|
|
|
|
for (; ; ) {
|
|
Pfn2 = MI_PFN_ELEMENT (Next);
|
|
Next = (PFN_NUMBER) Pfn2->OriginalPte.u.Long;
|
|
if (Page == Next) {
|
|
Pfn2->OriginalPte.u.Long = Pfn->OriginalPte.u.Long;
|
|
if ((PFN_NUMBER) Pfn->OriginalPte.u.Long == MM_EMPTY_LIST) {
|
|
MmFreePagesByColor[ListHead->ListName][Color].Blink = Pfn2;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
MmAvailablePages -= 1;
|
|
|
|
if (MmAvailablePages < MmMinimumFreePages) {
|
|
|
|
|
|
// Obtain free pages.
|
|
|
|
|
|
MiObtainFreePages();
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
|
|
ULONG
|
|
FASTCALL
|
|
MiEnsureAvailablePageOrWait (
|
|
IN PEPROCESS Process,
|
|
IN PVOID VirtualAddress
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This procedure ensures that a physical page is available on
|
|
the zeroed, free or standby list such that the next call the remove a
|
|
page absolutely will not block. This is necessary as blocking would
|
|
require a wait which could cause a deadlock condition.
|
|
|
|
If a page is available the function returns immediately with a value
|
|
of FALSE indicating no wait operation was performed. If no physical
|
|
page is available, the thread enters a wait state and the function
|
|
returns the value TRUE when the wait operation completes.
|
|
|
|
Arguments:
|
|
|
|
Process - Supplies a pointer to the current process if, and only if,
|
|
the working set mutex is held currently held and should
|
|
be released if a wait operation is issued. Supplies
|
|
the value NULL otherwise.
|
|
|
|
VirtualAddress - Supplies the virtual address for the faulting page.
|
|
If the value is NULL, the page is treated as a
|
|
user mode address.
|
|
|
|
Return Value:
|
|
|
|
FALSE - if a page was immediately available.
|
|
TRUE - if a wait operation occurred before a page became available.
|
|
|
|
|
|
Environment:
|
|
|
|
Must be holding the PFN database mutex with APCs disabled.
|
|
|
|
--*/
|
|
|
|
{
|
|
PVOID Event;
|
|
NTSTATUS Status;
|
|
KIRQL OldIrql;
|
|
KIRQL Ignore;
|
|
ULONG Limit;
|
|
ULONG Relock;
|
|
PFN_NUMBER StrandedPages;
|
|
LOGICAL WsHeldSafe;
|
|
PMMPFN Pfn1;
|
|
PMMPFN EndPfn;
|
|
LARGE_INTEGER WaitBegin;
|
|
LARGE_INTEGER WaitEnd;
|
|
|
|
MM_PFN_LOCK_ASSERT();
|
|
|
|
if (MmAvailablePages >= MM_HIGH_LIMIT) {
|
|
|
|
|
|
// Pages are available.
|
|
|
|
|
|
return FALSE;
|
|
}
|
|
|
|
|
|
// If this fault is for paged pool (or pagable kernel space,
|
|
// including page table pages), let it use the last page.
|
|
|
|
|
|
#if defined(_IA64_)
|
|
if (MI_IS_SYSTEM_ADDRESS(VirtualAddress) ||
|
|
(MI_IS_HYPER_SPACE_ADDRESS(VirtualAddress))) {
|
|
#else
|
|
if (((PMMPTE)VirtualAddress > MiGetPteAddress(HYPER_SPACE)) ||
|
|
((VirtualAddress > MM_HIGHEST_USER_ADDRESS) &&
|
|
(VirtualAddress < (PVOID)PTE_BASE))) {
|
|
#endif
|
|
|
|
|
|
// This fault is in the system, use 1 page as the limit.
|
|
|
|
|
|
if (MmAvailablePages >= MM_LOW_LIMIT) {
|
|
|
|
|
|
// Pages are available.
|
|
|
|
|
|
return FALSE;
|
|
}
|
|
|
|
Limit = MM_LOW_LIMIT;
|
|
Event = (PVOID)&MmAvailablePagesEvent;
|
|
} else {
|
|
Limit = MM_HIGH_LIMIT;
|
|
Event = (PVOID)&MmAvailablePagesEventHigh;
|
|
}
|
|
|
|
while (MmAvailablePages < Limit) {
|
|
KeClearEvent ((PKEVENT)Event);
|
|
|
|
UNLOCK_PFN (APC_LEVEL);
|
|
|
|
if (Process == HYDRA_PROCESS) {
|
|
UNLOCK_SESSION_SPACE_WS (APC_LEVEL);
|
|
}
|
|
else if (Process != NULL) {
|
|
|
|
|
|
// The working set lock may have been acquired safely or unsafely
|
|
// by our caller. Handle both cases here and below.
|
|
|
|
|
|
UNLOCK_WS_REGARDLESS (Process, WsHeldSafe);
|
|
}
|
|
else {
|
|
Relock = FALSE;
|
|
if (MmSystemLockOwner == PsGetCurrentThread()) {
|
|
UNLOCK_SYSTEM_WS (APC_LEVEL);
|
|
Relock = TRUE;
|
|
}
|
|
}
|
|
|
|
KiQueryInterruptTime(&WaitBegin);
|
|
|
|
|
|
// Wait 7 minutes for pages to become available.
|
|
|
|
|
|
Status = KeWaitForSingleObject(Event,
|
|
WrFreePage,
|
|
KernelMode,
|
|
FALSE,
|
|
(PLARGE_INTEGER)&MmSevenMinutes);
|
|
|
|
if (Status == STATUS_TIMEOUT) {
|
|
|
|
KiQueryInterruptTime(&WaitEnd);
|
|
|
|
|
|
// See how many transition pages have nonzero reference counts as
|
|
// these indicate drivers that aren't unlocking the pages in their
|
|
// MDLs.
|
|
|
|
|
|
Limit = 0;
|
|
StrandedPages = 0;
|
|
|
|
do {
|
|
|
|
Pfn1 = MI_PFN_ELEMENT (MmPhysicalMemoryBlock->Run[Limit].BasePage);
|
|
EndPfn = Pfn1 + MmPhysicalMemoryBlock->Run[Limit].PageCount;
|
|
|
|
while (Pfn1 < EndPfn) {
|
|
if ((Pfn1->u3.e1.PageLocation == TransitionPage) &&
|
|
(Pfn1->u3.e2.ReferenceCount != 0)) {
|
|
StrandedPages += 1;
|
|
}
|
|
Pfn1 += 1;
|
|
}
|
|
Limit += 1;
|
|
|
|
} while (Limit != MmPhysicalMemoryBlock->NumberOfRuns);
|
|
|
|
|
|
// This bugcheck can occur for the following reasons:
|
|
|
|
// A driver has blocked, deadlocking the modified or mapped
|
|
// page writers. Examples of this include mutex deadlocks or
|
|
// accesses to paged out memory in filesystem drivers, filter
|
|
// drivers, etc. This indicates a driver bug.
|
|
|
|
// The storage driver(s) are not processing requests. Examples
|
|
// of this are stranded queues, non-responding drives, etc. This
|
|
// indicates a driver bug.
|
|
|
|
// Not enough pool is available for the storage stack to write out
|
|
// modified pages. This indicates a driver bug.
|
|
|
|
// A high priority realtime thread has starved the balance set
|
|
// manager from trimming pages and/or starved the modified writer
|
|
// from writing them out. This indicates a bug in the component
|
|
// that created this thread.
|
|
|
|
// All the processes have been trimmed to their minimums and all
|
|
// modified pages written, but still no memory is available. The
|
|
// freed memory must be stuck in transition pages with non-zero
|
|
// reference counts - thus they cannot be put on the freelist.
|
|
// A driver is neglecting to unlock the pages preventing the
|
|
// reference counts from going to zero which would free the pages.
|
|
// This may be due to transfers that never finish and the driver
|
|
// never aborts or other driver bugs.
|
|
|
|
|
|
KeBugCheckEx (NO_PAGES_AVAILABLE,
|
|
MmModifiedPageListHead.Total,
|
|
MmTotalPagesForPagingFile,
|
|
(MmMaximumNonPagedPoolInBytes >> PAGE_SHIFT) - MmAllocatedNonPagedPool,
|
|
StrandedPages);
|
|
|
|
if (!KdDebuggerNotPresent) {
|
|
DbgPrint ("MmEnsureAvailablePageOrWait: 7 min timeout %x %x %x %x\n", WaitEnd.HighPart, WaitEnd.LowPart, WaitBegin.HighPart, WaitBegin.LowPart);
|
|
DbgBreakPoint ();
|
|
}
|
|
}
|
|
|
|
if (Process == HYDRA_PROCESS) {
|
|
LOCK_SESSION_SPACE_WS (Ignore);
|
|
}
|
|
else if (Process != NULL) {
|
|
|
|
|
|
// The working set lock may have been acquired safely or unsafely
|
|
// by our caller. Reacquire it in the same manner our caller did.
|
|
|
|
|
|
LOCK_WS_REGARDLESS (Process, WsHeldSafe);
|
|
}
|
|
else {
|
|
if (Relock) {
|
|
LOCK_SYSTEM_WS (Ignore);
|
|
}
|
|
}
|
|
|
|
LOCK_PFN (OldIrql);
|
|
}
|
|
|
|
return TRUE;
|
|
}
|
|
|
|
|
|
PFN_NUMBER //PageFrameIndex
|
|
FASTCALL
|
|
MiRemoveZeroPage (
|
|
IN ULONG PageColor
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This procedure removes a zero page from either the zeroed, free
|
|
or standby lists (in that order). If no pages exist on the zeroed
|
|
or free list a transition page is removed from the standby list
|
|
and the PTE (may be a prototype PTE) which refers to this page is
|
|
changed from transition back to its original contents.
|
|
|
|
If the page is not obtained from the zeroed list, it is zeroed.
|
|
|
|
Arguments:
|
|
|
|
PageColor - Supplies the page color for which this page is destined.
|
|
This is used for checking virtual address alignments to
|
|
determine if the D cache needs flushing before the page
|
|
can be reused.
|
|
|
|
Return Value:
|
|
|
|
The physical page number removed from the specified list.
|
|
|
|
Environment:
|
|
|
|
Must be holding the PFN database mutex with APCs disabled.
|
|
|
|
--*/
|
|
|
|
{
|
|
PFN_NUMBER Page;
|
|
PMMPFN Pfn1;
|
|
ULONG Color;
|
|
|
|
MM_PFN_LOCK_ASSERT();
|
|
ASSERT(MmAvailablePages != 0);
|
|
|
|
|
|
// Attempt to remove a page from the zeroed page list. If a page
|
|
// is available, then remove it and return its page frame index.
|
|
// Otherwise, attempt to remove a page from the free page list or
|
|
// the standby list.
|
|
|
|
// N.B. It is not necessary to change page colors even if the old
|
|
// color is not equal to the new color. The zero page thread
|
|
// ensures that all zeroed pages are removed from all caches.
|
|
|
|
|
|
if (MmFreePagesByColor[ZeroedPageList][PageColor].Flink != MM_EMPTY_LIST) {
|
|
|
|
|
|
// Remove the first entry on the zeroed by color list.
|
|
|
|
|
|
Page = MmFreePagesByColor[ZeroedPageList][PageColor].Flink;
|
|
|
|
#if DBG
|
|
Pfn1 = MI_PFN_ELEMENT(Page);
|
|
ASSERT (Pfn1->u3.e1.PageLocation == ZeroedPageList);
|
|
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
#endif
|
|
|
|
MiRemovePageByColor (Page, PageColor);
|
|
|
|
#if DBG
|
|
ASSERT (Pfn1->u3.e1.PageColor == (PageColor & MM_COLOR_MASK));
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
ASSERT (Pfn1->u2.ShareCount == 0);
|
|
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
#endif
|
|
return Page;
|
|
|
|
}
|
|
|
|
|
|
// No previously zeroed page with the specified secondary color exists.
|
|
// Try a zeroed page of the primary color.
|
|
|
|
|
|
if (MmZeroedPageListHead.Flink != MM_EMPTY_LIST) {
|
|
Page = MmZeroedPageListHead.Flink;
|
|
#if DBG
|
|
Pfn1 = MI_PFN_ELEMENT(Page);
|
|
ASSERT (Pfn1->u3.e1.PageLocation == ZeroedPageList);
|
|
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
#endif
|
|
Color = MI_GET_SECONDARY_COLOR (Page, MI_PFN_ELEMENT(Page));
|
|
MiRemovePageByColor (Page, Color);
|
|
#if DBG
|
|
Pfn1 = MI_PFN_ELEMENT(Page);
|
|
ASSERT (Pfn1->u3.e1.PageColor == (PageColor & MM_COLOR_MASK));
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
ASSERT (Pfn1->u2.ShareCount == 0);
|
|
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
#endif
|
|
return Page;
|
|
}
|
|
|
|
|
|
// No zeroed page of the primary color exists, try a free page of the
|
|
// secondary color.
|
|
|
|
|
|
if (MmFreePagesByColor[FreePageList][PageColor].Flink != MM_EMPTY_LIST) {
|
|
|
|
|
|
// Remove the first entry on the free list by color.
|
|
|
|
|
|
Page = MmFreePagesByColor[FreePageList][PageColor].Flink;
|
|
|
|
#if DBG
|
|
Pfn1 = MI_PFN_ELEMENT(Page);
|
|
ASSERT (Pfn1->u3.e1.PageLocation == FreePageList);
|
|
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
#endif
|
|
|
|
MiRemovePageByColor (Page, PageColor);
|
|
#if DBG
|
|
Pfn1 = MI_PFN_ELEMENT(Page);
|
|
ASSERT (Pfn1->u3.e1.PageColor == (PageColor & MM_COLOR_MASK));
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
ASSERT (Pfn1->u2.ShareCount == 0);
|
|
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
#endif
|
|
goto ZeroPage;
|
|
}
|
|
|
|
if (MmFreePageListHead.Flink != MM_EMPTY_LIST) {
|
|
Page = MmFreePageListHead.Flink;
|
|
|
|
Color = MI_GET_SECONDARY_COLOR (Page, MI_PFN_ELEMENT(Page));
|
|
MiRemovePageByColor (Page, Color);
|
|
#if DBG
|
|
Pfn1 = MI_PFN_ELEMENT(Page);
|
|
ASSERT (Pfn1->u3.e1.PageColor == (PageColor & MM_COLOR_MASK));
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
ASSERT (Pfn1->u2.ShareCount == 0);
|
|
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
#endif
|
|
goto ZeroPage;
|
|
}
|
|
|
|
ASSERT (MmZeroedPageListHead.Total == 0);
|
|
ASSERT (MmFreePageListHead.Total == 0);
|
|
|
|
if (MmZeroedPageListHead.Total != 0) {
|
|
|
|
Page = MiRemovePageFromList(&MmZeroedPageListHead);
|
|
MI_CHECK_PAGE_ALIGNMENT(Page, PageColor & MM_COLOR_MASK);
|
|
|
|
} else {
|
|
|
|
|
|
// Attempt to remove a page from the free list. If a page is
|
|
// available, then remove it. Otherwise, attempt to remove a
|
|
// page from the standby list.
|
|
|
|
|
|
if (MmFreePageListHead.Total != 0) {
|
|
Page = MiRemovePageFromList(&MmFreePageListHead);
|
|
ASSERT ((MI_PFN_ELEMENT(Page))->PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
} else {
|
|
|
|
|
|
// Remove a page from the standby list and restore the original
|
|
// contents of the PTE to free the last reference to the physical
|
|
// page.
|
|
|
|
|
|
ASSERT (MmStandbyPageListHead.Total != 0);
|
|
|
|
Page = MiRemovePageFromList(&MmStandbyPageListHead);
|
|
ASSERT ((MI_PFN_ELEMENT(Page))->PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
}
|
|
|
|
|
|
// Zero the page removed from the free or standby list.
|
|
|
|
|
|
ZeroPage:
|
|
|
|
Pfn1 = MI_PFN_ELEMENT(Page);
|
|
|
|
#if defined(_ALPHA_)
|
|
HalZeroPage((PVOID)ULongToPtr((PageColor & MM_COLOR_MASK) << PAGE_SHIFT),
|
|
(PVOID)ULongToPtr((Pfn1->u3.e1.PageColor) << PAGE_SHIFT),
|
|
Page);
|
|
#else
|
|
MiZeroPhysicalPage (Page, 0);
|
|
#endif
|
|
|
|
|
|
// Note the stamping must occur after the page is zeroed.
|
|
|
|
|
|
MI_BARRIER_STAMP_ZEROED_PAGE (&Pfn1->PteFrame);
|
|
|
|
Pfn1->u3.e1.PageColor = PageColor & MM_COLOR_MASK;
|
|
|
|
}
|
|
|
|
#if DBG
|
|
Pfn1 = MI_PFN_ELEMENT (Page);
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
ASSERT (Pfn1->u2.ShareCount == 0);
|
|
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
#endif
|
|
|
|
return Page;
|
|
}
|
|
|
|
PFN_NUMBER //PageFrameIndex
|
|
FASTCALL
|
|
MiRemoveAnyPage (
|
|
IN ULONG PageColor
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This procedure removes a page from either the free, zeroed,
|
|
or standby lists (in that order). If no pages exist on the zeroed
|
|
or free list a transition page is removed from the standby list
|
|
and the PTE (may be a prototype PTE) which refers to this page is
|
|
changed from transition back to its original contents.
|
|
|
|
Note pages MUST exist to satisfy this request. The caller ensures this
|
|
by first calling MiEnsureAvailablePageOrWait.
|
|
|
|
Arguments:
|
|
|
|
PageColor - Supplies the page color for which this page is destined.
|
|
This is used for checking virtual address alignments to
|
|
determine if the D cache needs flushing before the page
|
|
can be reused.
|
|
|
|
Return Value:
|
|
|
|
The physical page number removed from the specified list.
|
|
|
|
Environment:
|
|
|
|
Must be holding the PFN database mutex with APCs disabled.
|
|
|
|
--*/
|
|
|
|
{
|
|
PFN_NUMBER Page;
|
|
PMMPFN Pfn1;
|
|
ULONG Color;
|
|
|
|
MM_PFN_LOCK_ASSERT();
|
|
ASSERT(MmAvailablePages != 0);
|
|
|
|
|
|
// Check the free page list, and if a page is available
|
|
// remove it and return its value.
|
|
|
|
|
|
if (MmFreePagesByColor[FreePageList][PageColor].Flink != MM_EMPTY_LIST) {
|
|
|
|
|
|
// Remove the first entry on the free by color list.
|
|
|
|
|
|
Page = MmFreePagesByColor[FreePageList][PageColor].Flink;
|
|
#if DBG
|
|
Pfn1 = MI_PFN_ELEMENT(Page);
|
|
ASSERT (Pfn1->u3.e1.PageLocation == FreePageList);
|
|
ASSERT (Pfn1->u3.e1.PageColor == (PageColor & MM_COLOR_MASK));
|
|
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
#endif
|
|
MiRemovePageByColor (Page, PageColor);
|
|
#if DBG
|
|
ASSERT (Pfn1->u3.e1.PageColor == (PageColor & MM_COLOR_MASK));
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
ASSERT (Pfn1->u2.ShareCount == 0);
|
|
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
#endif
|
|
return Page;
|
|
|
|
}
|
|
|
|
if (MmFreePagesByColor[ZeroedPageList][PageColor].Flink
|
|
!= MM_EMPTY_LIST) {
|
|
|
|
|
|
// Remove the first entry on the zeroed by color list.
|
|
|
|
|
|
Page = MmFreePagesByColor[ZeroedPageList][PageColor].Flink;
|
|
#if DBG
|
|
Pfn1 = MI_PFN_ELEMENT(Page);
|
|
ASSERT (Pfn1->u3.e1.PageColor == (PageColor & MM_COLOR_MASK));
|
|
ASSERT (Pfn1->u3.e1.PageLocation == ZeroedPageList);
|
|
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
#endif
|
|
|
|
MiRemovePageByColor (Page, PageColor);
|
|
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
return Page;
|
|
}
|
|
|
|
|
|
// Try the free page list by primary color.
|
|
|
|
|
|
if (MmFreePageListHead.Flink != MM_EMPTY_LIST) {
|
|
Page = MmFreePageListHead.Flink;
|
|
Color = MI_GET_SECONDARY_COLOR (Page, MI_PFN_ELEMENT(Page));
|
|
|
|
#if DBG
|
|
Pfn1 = MI_PFN_ELEMENT(Page);
|
|
ASSERT (Pfn1->u3.e1.PageColor == (PageColor & MM_COLOR_MASK));
|
|
ASSERT (Pfn1->u3.e1.PageLocation == FreePageList);
|
|
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
#endif
|
|
MiRemovePageByColor (Page, Color);
|
|
#if DBG
|
|
ASSERT (Pfn1->u3.e1.PageColor == (PageColor & MM_COLOR_MASK));
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
ASSERT (Pfn1->u2.ShareCount == 0);
|
|
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
#endif
|
|
return Page;
|
|
|
|
}
|
|
|
|
if (MmZeroedPageListHead.Flink != MM_EMPTY_LIST) {
|
|
Page = MmZeroedPageListHead.Flink;
|
|
Color = MI_GET_SECONDARY_COLOR (Page, MI_PFN_ELEMENT(Page));
|
|
MiRemovePageByColor (Page, Color);
|
|
#if DBG
|
|
Pfn1 = MI_PFN_ELEMENT(Page);
|
|
ASSERT (Pfn1->u3.e1.PageColor == (PageColor & MM_COLOR_MASK));
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
ASSERT (Pfn1->u2.ShareCount == 0);
|
|
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
#endif
|
|
return Page;
|
|
}
|
|
|
|
if (MmFreePageListHead.Total != 0) {
|
|
|
|
Page = MiRemovePageFromList(&MmFreePageListHead);
|
|
ASSERT ((MI_PFN_ELEMENT(Page))->PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
|
|
} else {
|
|
|
|
|
|
// Check the zeroed page list, and if a page is available
|
|
// remove it and return its value.
|
|
|
|
|
|
if (MmZeroedPageListHead.Total != 0) {
|
|
|
|
Page = MiRemovePageFromList(&MmZeroedPageListHead);
|
|
ASSERT ((MI_PFN_ELEMENT(Page))->PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
|
|
} else {
|
|
|
|
|
|
// No pages exist on the free or zeroed list, use the
|
|
// standby list.
|
|
|
|
|
|
ASSERT(MmStandbyPageListHead.Total != 0);
|
|
|
|
Page = MiRemovePageFromList(&MmStandbyPageListHead);
|
|
ASSERT ((MI_PFN_ELEMENT(Page))->PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
|
}
|
|
}
|
|
|
|
MI_CHECK_PAGE_ALIGNMENT(Page, PageColor & MM_COLOR_MASK);
|
|
#if DBG
|
|
Pfn1 = MI_PFN_ELEMENT (Page);
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
ASSERT (Pfn1->u2.ShareCount == 0);
|
|
#endif
|
|
return Page;
|
|
}
|
|
|
|
|
|
VOID
|
|
MiRemovePageByColor (
|
|
IN PFN_NUMBER Page,
|
|
IN ULONG Color
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This procedure removes a page from the middle of the free or
|
|
zeroed page list.
|
|
|
|
Arguments:
|
|
|
|
PageFrameIndex - Supplies the physical page number to unlink from the
|
|
list.
|
|
|
|
Return Value:
|
|
|
|
none.
|
|
|
|
Environment:
|
|
|
|
Must be holding the PFN database mutex with APCs disabled.
|
|
|
|
--*/
|
|
|
|
{
|
|
PMMPFNLIST ListHead;
|
|
PMMPFNLIST PrimaryListHead;
|
|
PFN_NUMBER Previous;
|
|
PFN_NUMBER Next;
|
|
PMMPFN Pfn1;
|
|
PMMPFN Pfn2;
|
|
ULONG PrimaryColor;
|
|
|
|
MM_PFN_LOCK_ASSERT();
|
|
|
|
Pfn1 = MI_PFN_ELEMENT (Page);
|
|
PrimaryColor = Pfn1->u3.e1.PageColor;
|
|
|
|
PERFINFO_REMOVEPAGE(Page, PERFINFO_LOG_TYPE_REMOVEPAGEBYCOLOR);
|
|
|
|
ListHead = MmPageLocationList[Pfn1->u3.e1.PageLocation];
|
|
|
|
ListHead->Total -= 1;
|
|
|
|
PrimaryListHead = ListHead;
|
|
|
|
#if PFN_CONSISTENCY
|
|
if (MmFreePagesByColor[PrimaryListHead->ListName][Color].Flink != Page) {
|
|
|
|
KeBugCheckEx (PFN_LIST_CORRUPT,
|
|
0x9A,
|
|
Page,
|
|
Color,
|
|
(PFN_NUMBER)&MmFreePagesByColor[PrimaryListHead->ListName][Color].Flink);
|
|
}
|
|
#endif
|
|
|
|
Next = Pfn1->u1.Flink;
|
|
Pfn1->u1.Flink = 0; // Assumes Flink width is >= WsIndex width
|
|
Previous = Pfn1->u2.Blink;
|
|
Pfn1->u2.Blink = 0;
|
|
|
|
if (Next == MM_EMPTY_LIST) {
|
|
PrimaryListHead->Blink = Previous;
|
|
} else {
|
|
Pfn2 = MI_PFN_ELEMENT(Next);
|
|
Pfn2->u2.Blink = Previous;
|
|
}
|
|
|
|
if (Previous == MM_EMPTY_LIST) {
|
|
PrimaryListHead->Flink = Next;
|
|
} else {
|
|
Pfn2 = MI_PFN_ELEMENT(Previous);
|
|
Pfn2->u1.Flink = Next;
|
|
}
|
|
|
|
|
|
// Zero the flags longword, but keep the color information.
|
|
|
|
|
|
ASSERT (Pfn1->u3.e1.RemovalRequested == 0);
|
|
Pfn1->u3.e2.ShortFlags = 0;
|
|
Pfn1->u3.e1.PageColor = PrimaryColor;
|
|
|
|
|
|
// Update the color lists.
|
|
|
|
|
|
MmFreePagesByColor[ListHead->ListName][Color].Flink =
|
|
(PFN_NUMBER) Pfn1->OriginalPte.u.Long;
|
|
|
|
|
|
// Note that we now have one less page available.
|
|
|
|
|
|
MmAvailablePages -= 1;
|
|
|
|
if (MmAvailablePages < MmMinimumFreePages) {
|
|
|
|
|
|
// Obtain free pages.
|
|
|
|
|
|
MiObtainFreePages();
|
|
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiInsertFrontModifiedNoWrite (
|
|
IN PFN_NUMBER PageFrameIndex
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This procedure inserts a page at the FRONT of the modified no
|
|
write list.
|
|
|
|
Arguments:
|
|
|
|
PageFrameIndex - Supplies the physical page number to insert in the
|
|
list.
|
|
|
|
Return Value:
|
|
|
|
none.
|
|
|
|
Environment:
|
|
|
|
Must be holding the PFN database mutex with APCs disabled.
|
|
|
|
--*/
|
|
|
|
{
|
|
PFN_NUMBER first;
|
|
PMMPFN Pfn1;
|
|
PMMPFN Pfn2;
|
|
|
|
MM_PFN_LOCK_ASSERT();
|
|
ASSERT ((PageFrameIndex != 0) && (PageFrameIndex <= MmHighestPhysicalPage) &&
|
|
(PageFrameIndex >= MmLowestPhysicalPage));
|
|
|
|
|
|
// Check to ensure the reference count for the page
|
|
// is zero.
|
|
|
|
|
|
Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
|
|
|
|
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
|
|
|
MmModifiedNoWritePageListHead.Total += 1; // One more page on the list.
|
|
|
|
MI_TALLY_TRANSITION_PAGE_ADDITION (Pfn1);
|
|
|
|
first = MmModifiedNoWritePageListHead.Flink;
|
|
if (first == MM_EMPTY_LIST) {
|
|
|
|
|
|
// List is empty add the page to the ListHead.
|
|
|
|
|
|
MmModifiedNoWritePageListHead.Blink = PageFrameIndex;
|
|
} else {
|
|
Pfn2 = MI_PFN_ELEMENT (first);
|
|
Pfn2->u2.Blink = PageFrameIndex;
|
|
}
|
|
|
|
MmModifiedNoWritePageListHead.Flink = PageFrameIndex;
|
|
Pfn1->u1.Flink = first;
|
|
Pfn1->u2.Blink = MM_EMPTY_LIST;
|
|
Pfn1->u3.e1.PageLocation = ModifiedNoWritePageList;
|
|
return;
|
|
}
|