Microsoft/Windows2000
Microsoft/Windows2000
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
master
Windows2000/private/ntos/mm/pfnlist.c

1750 lines
45 KiB
C
Raw Permalink Normal View History

First commit
2001-01-01 00:00:00 +01:00
/*++
Copyright (c) 1989 Microsoft Corporation
Module Name:
pfnlist.c
Abstract:
This module contains the routines to manipulate pages within the
Page Frame Database.
Author:
Lou Perazzoli (loup) 4-Apr-1989
Landy Wang (landyw) 02-June-1997
Revision History:
--*/
#include "mi.h"
#define MM_LOW_LIMIT 2
#define MM_HIGH_LIMIT 19
KEVENT MmAvailablePagesEventHigh;
ULONG MmTransitionPrivatePages;
ULONG MmTransitionSharedPages;
#define MI_TALLY_TRANSITION_PAGE_ADDITION(Pfn) \
if (Pfn->u3.e1.PrototypePte) { \
MmTransitionSharedPages += 1; \
} \
else { \
MmTransitionPrivatePages += 1; \
} \
ASSERT (MmTransitionPrivatePages + MmTransitionSharedPages == MmStandbyPageListHead.Total + MmModifiedPageListHead.Total + MmModifiedNoWritePageListHead.Total);
#define MI_TALLY_TRANSITION_PAGE_REMOVAL(Pfn) \
if (Pfn->u3.e1.PrototypePte) { \
MmTransitionSharedPages -= 1; \
} \
else { \
MmTransitionPrivatePages -= 1; \
} \
ASSERT (MmTransitionPrivatePages + MmTransitionSharedPages == MmStandbyPageListHead.Total + MmModifiedPageListHead.Total + MmModifiedNoWritePageListHead.Total);
VOID
MiRemovePageByColor (
IN PFN_NUMBER Page,
IN ULONG PageColor
);
VOID
FASTCALL
MiInsertPageInList (
IN PMMPFNLIST ListHead,
IN PFN_NUMBER PageFrameIndex
)
/*++
Routine Description:
This procedure inserts a page at the end of the specified list (free,
standby, bad, zeroed, modified).
Arguments:
ListHead - Supplies the list of the list in which to insert the
specified physical page.
PageFrameIndex - Supplies the physical page number to insert in the
list.
Return Value:
none.
Environment:
Must be holding the PFN database mutex with APCs disabled.
--*/
{
PFN_NUMBER last;
PMMPFN Pfn1;
PMMPFN Pfn2;
ULONG Color;
MM_PFN_LOCK_ASSERT();
ASSERT ((PageFrameIndex != 0) &&
(PageFrameIndex <= MmHighestPhysicalPage) &&
(PageFrameIndex >= MmLowestPhysicalPage));
// Check to ensure the reference count for the page is zero.
Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
ASSERT (Pfn1->u3.e1.LockCharged == 0);
PERFINFO_INSERTINLIST(PageFrameIndex, ListHead);
#if DBG
if (MmDebug & MM_DBG_PAGE_REF_COUNT) {
PMMPTE PointerPte;
KIRQL OldIrql = 99;
if ((ListHead->ListName == StandbyPageList) ||
(ListHead->ListName == ModifiedPageList)) {
if ((Pfn1->u3.e1.PrototypePte == 1) &&
(MmIsAddressValid (Pfn1->PteAddress))) {
PointerPte = Pfn1->PteAddress;
} else {
// The page containing the prototype PTE is not valid,
// map the page into hyperspace and reference it that way.
PointerPte = MiMapPageInHyperSpace (Pfn1->PteFrame, &OldIrql);
PointerPte = (PMMPTE)((PCHAR)PointerPte +
MiGetByteOffset(Pfn1->PteAddress));
}
ASSERT ((MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (PointerPte) == PageFrameIndex) ||
(MI_GET_PAGE_FRAME_FROM_PTE (PointerPte) == PageFrameIndex));
ASSERT (PointerPte->u.Soft.Transition == 1);
ASSERT (PointerPte->u.Soft.Prototype == 0);
if (OldIrql != 99) {
MiUnmapPageInHyperSpace (OldIrql)
}
}
}
#endif
#if PFN_CONSISTENCY
if (ListHead == &MmFreePageListHead) {
if (Pfn1->u2.ShareCount != 0) {
KeBugCheckEx (PFN_LIST_CORRUPT,
0x91,
PageFrameIndex,
Pfn1->u2.ShareCount,
Pfn1->u3.e2.ReferenceCount);
}
}
else if (ListHead == &MmZeroedPageListHead) {
if (Pfn1->u2.ShareCount != 0) {
KeBugCheckEx (PFN_LIST_CORRUPT,
0x92,
PageFrameIndex,
Pfn1->u2.ShareCount,
Pfn1->u3.e2.ReferenceCount);
}
}
#endif
#if DBG
if ((ListHead->ListName == StandbyPageList) ||
(ListHead->ListName == ModifiedPageList)) {
if ((Pfn1->OriginalPte.u.Soft.Prototype == 0) &&
(Pfn1->OriginalPte.u.Soft.Transition == 1)) {
KeBugCheckEx (MEMORY_MANAGEMENT, 0x8888, 0,0,0);
}
}
#endif
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
ListHead->Total += 1; // One more page on the list.
// On MIPS R4000 modified pages destined for the paging file are
// kept on separate lists which group pages of the same color
// together
if (ListHead == &MmModifiedPageListHead) {
#if PFN_CONSISTENCY
if (Pfn1->u2.ShareCount != 0) {
KeBugCheckEx (PFN_LIST_CORRUPT,
0x90,
PageFrameIndex,
Pfn1->u2.ShareCount,
Pfn1->u3.e2.ReferenceCount);
}
#endif
if (Pfn1->OriginalPte.u.Soft.Prototype == 0) {
// This page is destined for the paging file (not
// a mapped file). Change the list head to the
// appropriate colored list head.
ListHead = &MmModifiedPageListByColor [Pfn1->u3.e1.PageColor];
ListHead->Total += 1;
MmTotalPagesForPagingFile += 1;
}
else {
// This page is destined for a mapped file (not
// the paging file). If there are no other pages currently
// destined for the mapped file, start our timer so that we can
// ensure that these pages make it to disk even if we don't pile
// up enough of them to trigger the modified page writer or need
// the memory. If we don't do this here, then for this scenario,
// only an orderly system shutdown will write them out (days,
// weeks, months or years later) and any power out in between
// means we'll have lost the data.
if (ListHead->Total - MmTotalPagesForPagingFile == 1) {
// Start the DPC timer because we're the first on the list.
if (MiTimerPending == FALSE) {
MiTimerPending = TRUE;
(VOID) KeSetTimerEx( &MiModifiedPageWriterTimer, MiModifiedPageLife, 0, &MiModifiedPageWriterTimerDpc );
}
}
}
}
else if ((Pfn1->u3.e1.RemovalRequested == 1) &&
(ListHead->ListName <= StandbyPageList)) {
ListHead->Total -= 1; // Undo previous increment
if (ListHead->ListName == StandbyPageList) {
Pfn1->u3.e1.PageLocation = StandbyPageList;
MiRestoreTransitionPte (PageFrameIndex);
}
ListHead = MmPageLocationList[BadPageList];
ListHead->Total += 1; // One more page on the list.
}
last = ListHead->Blink;
if (last == MM_EMPTY_LIST) {
// List is empty add the page to the ListHead.
ListHead->Flink = PageFrameIndex;
} else {
Pfn2 = MI_PFN_ELEMENT (last);
Pfn2->u1.Flink = PageFrameIndex;
}
ListHead->Blink = PageFrameIndex;
Pfn1->u1.Flink = MM_EMPTY_LIST;
Pfn1->u2.Blink = last;
Pfn1->u3.e1.PageLocation = ListHead->ListName;
// If the page was placed on the free, standby or zeroed list,
// update the count of usable pages in the system. If the count
// transitions from 0 to 1, the event associated with available
// pages should become true.
if (ListHead->ListName <= StandbyPageList) {
MmAvailablePages += 1;
// A page has just become available, check to see if the
// page wait events should be signalled.
if (MmAvailablePages == MM_LOW_LIMIT) {
KeSetEvent (&MmAvailablePagesEvent, 0, FALSE);
} else if (MmAvailablePages == MM_HIGH_LIMIT) {
KeSetEvent (&MmAvailablePagesEventHigh, 0, FALSE);
}
if (ListHead->ListName <= FreePageList) {
ASSERT (Pfn1->u3.e1.InPageError == 0);
// We are adding a page to the free or zeroed page list.
// Add the page to the end of the correct colored page list.
Color = MI_GET_SECONDARY_COLOR (PageFrameIndex, Pfn1);
ASSERT (Pfn1->u3.e1.PageColor == MI_GET_COLOR_FROM_SECONDARY(Color));
if (MmFreePagesByColor[ListHead->ListName][Color].Flink ==
MM_EMPTY_LIST) {
// This list is empty, add this as the first and last
// entry.
MmFreePagesByColor[ListHead->ListName][Color].Flink =
PageFrameIndex;
MmFreePagesByColor[ListHead->ListName][Color].Blink =
(PVOID)Pfn1;
} else {
Pfn2 = (PMMPFN)MmFreePagesByColor[ListHead->ListName][Color].Blink;
Pfn2->OriginalPte.u.Long = PageFrameIndex;
MmFreePagesByColor[ListHead->ListName][Color].Blink = (PVOID)Pfn1;
}
Pfn1->OriginalPte.u.Long = MM_EMPTY_LIST;
if (ListHead->ListName == ZeroedPageList) {
MI_BARRIER_STAMP_ZEROED_PAGE (&Pfn1->PteFrame);
}
}
else {
// Transition page list so tally it appropriately.
MI_TALLY_TRANSITION_PAGE_ADDITION (Pfn1);
}
if ((ListHead->ListName == FreePageList) &&
(MmFreePageListHead.Total >= MmMinimumFreePagesToZero) &&
(MmZeroingPageThreadActive == FALSE)) {
// There are enough pages on the free list, start
// the zeroing page thread.
MmZeroingPageThreadActive = TRUE;
KeSetEvent (&MmZeroingPageEvent, 0, FALSE);
}
return;
}
// Check to see if there are too many modified pages.
if (ListHead->ListName == ModifiedPageList) {
// Transition page list so tally it appropriately.
MI_TALLY_TRANSITION_PAGE_ADDITION (Pfn1);
if (Pfn1->OriginalPte.u.Soft.Prototype == 0) {
ASSERT (Pfn1->OriginalPte.u.Soft.PageFileHigh == 0);
}
PsGetCurrentProcess()->ModifiedPageCount += 1;
if (MmModifiedPageListHead.Total >= MmModifiedPageMaximum ) {
// Start the modified page writer.
KeSetEvent (&MmModifiedPageWriterEvent, 0, FALSE);
}
}
else if (ListHead->ListName == ModifiedNoWritePageList) {
MI_TALLY_TRANSITION_PAGE_ADDITION (Pfn1);
}
return;
}
VOID
FASTCALL
MiInsertStandbyListAtFront (
IN PFN_NUMBER PageFrameIndex
)
/*++
Routine Description:
This procedure inserts a page at the front of the standby list.
Arguments:
PageFrameIndex - Supplies the physical page number to insert in the
list.
Return Value:
none.
Environment:
Must be holding the PFN database mutex with APCs disabled.
--*/
{
PFN_NUMBER first;
IN PMMPFNLIST ListHead;
PMMPFN Pfn1;
PMMPFN Pfn2;
MM_PFN_LOCK_ASSERT();
ASSERT ((PageFrameIndex != 0) && (PageFrameIndex <= MmHighestPhysicalPage) &&
(PageFrameIndex >= MmLowestPhysicalPage));
// Check to ensure the reference count for the page
// is zero.
Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
PERFINFO_INSERT_FRONT_STANDBY(PageFrameIndex);
#if DBG
if (MmDebug & MM_DBG_PAGE_REF_COUNT) {
PMMPTE PointerPte;
KIRQL OldIrql = 99;
if ((Pfn1->u3.e1.PrototypePte == 1) &&
(MmIsAddressValid (Pfn1->PteAddress))) {
PointerPte = Pfn1->PteAddress;
} else {
// The page containing the prototype PTE is not valid,
// map the page into hyperspace and reference it that way.
PointerPte = MiMapPageInHyperSpace (Pfn1->PteFrame, &OldIrql);
PointerPte = (PMMPTE)((PCHAR)PointerPte +
MiGetByteOffset(Pfn1->PteAddress));
}
ASSERT ((MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (PointerPte) == PageFrameIndex) ||
(MI_GET_PAGE_FRAME_FROM_PTE (PointerPte) == PageFrameIndex));
ASSERT (PointerPte->u.Soft.Transition == 1);
ASSERT (PointerPte->u.Soft.Prototype == 0);
if (OldIrql != 99) {
MiUnmapPageInHyperSpace (OldIrql)
}
}
if ((Pfn1->OriginalPte.u.Soft.Prototype == 0) &&
(Pfn1->OriginalPte.u.Soft.Transition == 1)) {
KeBugCheckEx (MEMORY_MANAGEMENT, 0x8889, 0,0,0);
}
#endif
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
MmTransitionSharedPages += 1;
MmStandbyPageListHead.Total += 1; // One more page on the list.
ASSERT (MmTransitionPrivatePages + MmTransitionSharedPages == MmStandbyPageListHead.Total + MmModifiedPageListHead.Total + MmModifiedNoWritePageListHead.Total);
ListHead = &MmStandbyPageListHead;
first = ListHead->Flink;
if (first == MM_EMPTY_LIST) {
// List is empty add the page to the ListHead.
ListHead->Blink = PageFrameIndex;
} else {
Pfn2 = MI_PFN_ELEMENT (first);
Pfn2->u2.Blink = PageFrameIndex;
}
ListHead->Flink = PageFrameIndex;
Pfn1->u2.Blink = MM_EMPTY_LIST;
Pfn1->u1.Flink = first;
Pfn1->u3.e1.PageLocation = StandbyPageList;
// If the page was placed on the free, standby or zeroed list,
// update the count of usable pages in the system. If the count
// transitions from 0 to 1, the event associated with available
// pages should become true.
MmAvailablePages += 1;
// A page has just become available, check to see if the
// page wait events should be signalled.
if (MmAvailablePages == MM_LOW_LIMIT) {
KeSetEvent (&MmAvailablePagesEvent, 0, FALSE);
} else if (MmAvailablePages == MM_HIGH_LIMIT) {
KeSetEvent (&MmAvailablePagesEventHigh, 0, FALSE);
}
return;
}
PFN_NUMBER //PageFrameIndex
FASTCALL
MiRemovePageFromList (
IN PMMPFNLIST ListHead
)
/*++
Routine Description:
This procedure removes a page from the head of the specified list (free,
standby, zeroed, modified).
This routine clears the flags word in the PFN database, hence the
PFN information for this page must be initialized.
Arguments:
ListHead - Supplies the list of the list in which to remove the
specified physical page.
Return Value:
The physical page number removed from the specified list.
Environment:
Must be holding the PFN database mutex with APCs disabled.
--*/
{
PFN_NUMBER PageFrameIndex;
PMMPFN Pfn1;
PMMPFN Pfn2;
ULONG Color;
MM_PFN_LOCK_ASSERT();
// If the specified list is empty return MM_EMPTY_LIST.
if (ListHead->Total == 0) {
KdPrint(("MM:Attempting to remove page from empty list\n"));
KeBugCheckEx (PFN_LIST_CORRUPT, 1, (ULONG_PTR)ListHead, MmAvailablePages, 0);
return 0;
}
ASSERT (ListHead->ListName != ModifiedPageList);
// Decrement the count of pages on the list and remove the first
// page from the list.
ListHead->Total -= 1;
PageFrameIndex = ListHead->Flink;
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
PERFINFO_REMOVEPAGE(PageFrameIndex, PERFINFO_LOG_TYPE_REMOVEPAGEFROMLIST);
ListHead->Flink = Pfn1->u1.Flink;
// Zero the flink and blink in the pfn database element.
Pfn1->u1.Flink = 0; // Assumes Flink width is >= WsIndex width
Pfn1->u2.Blink = 0;
// If the last page was removed (the ListHead->Flink is now
// MM_EMPTY_LIST) make the listhead->Blink MM_EMPTY_LIST as well.
if (ListHead->Flink == MM_EMPTY_LIST) {
ListHead->Blink = MM_EMPTY_LIST;
} else {
// Make the PFN element point to MM_EMPTY_LIST signifying this
// is the last page in the list.
Pfn2 = MI_PFN_ELEMENT (ListHead->Flink);
Pfn2->u2.Blink = MM_EMPTY_LIST;
}
// Check to see if we now have one less page available.
if (ListHead->ListName <= StandbyPageList) {
MmAvailablePages -= 1;
if (ListHead->ListName == StandbyPageList) {
// This page is currently in transition, restore the PTE to
// its original contents so this page can be reused.
MI_TALLY_TRANSITION_PAGE_REMOVAL (Pfn1);
MiRestoreTransitionPte (PageFrameIndex);
}
if (MmAvailablePages < MmMinimumFreePages) {
// Obtain free pages.
MiObtainFreePages();
}
}
ASSERT ((PageFrameIndex != 0) &&
(PageFrameIndex <= MmHighestPhysicalPage) &&
(PageFrameIndex >= MmLowestPhysicalPage));
// Zero the PFN flags longword.
Color = Pfn1->u3.e1.PageColor;
ASSERT (Pfn1->u3.e1.RemovalRequested == 0);
Pfn1->u3.e2.ShortFlags = 0;
Pfn1->u3.e1.PageColor = Color;
Color = MI_GET_SECONDARY_COLOR (PageFrameIndex, Pfn1);
if (ListHead->ListName <= FreePageList) {
// Update the color lists.
ASSERT (MmFreePagesByColor[ListHead->ListName][Color].Flink == PageFrameIndex);
MmFreePagesByColor[ListHead->ListName][Color].Flink =
(PFN_NUMBER) Pfn1->OriginalPte.u.Long;
}
return PageFrameIndex;
}
VOID
FASTCALL
MiUnlinkPageFromList (
IN PMMPFN Pfn
)
/*++
Routine Description:
This procedure removes a page from the middle of a list. This is
designed for the faulting of transition pages from the standby and
modified list and making them active and valid again.
Arguments:
Pfn - Supplies a pointer to the PFN database element for the physical
page to remove from the list.
Return Value:
none.
Environment:
Must be holding the PFN database mutex with APCs disabled.
--*/
{
PMMPFNLIST ListHead;
PFN_NUMBER Previous;
PFN_NUMBER Next;
PMMPFN Pfn2;
MM_PFN_LOCK_ASSERT();
PERFINFO_UNLINKPAGE((ULONG_PTR)(Pfn - MmPfnDatabase), Pfn->u3.e1.PageLocation);
// Page not on standby or modified list, check to see if the
// page is currently being written by the modified page
// writer, if so, just return this page. The reference
// count for the page will be incremented, so when the modified
// page write completes, the page will not be put back on
// the list, rather, it will remain active and valid.
if (Pfn->u3.e2.ReferenceCount > 0) {
// The page was not on any "transition lists", check to see
// if this has I/O in progress.
if (Pfn->u2.ShareCount == 0) {
#if DBG
if (MmDebug & MM_DBG_PAGE_IN_LIST) {
DbgPrint("unlinking page not in list...\n");
MiFormatPfn(Pfn);
}
#endif
return;
}
KdPrint(("MM:attempt to remove page from wrong page list\n"));
KeBugCheckEx (PFN_LIST_CORRUPT,
2,
Pfn - MmPfnDatabase,
MmHighestPhysicalPage,
Pfn->u3.e2.ReferenceCount);
return;
}
ListHead = MmPageLocationList[Pfn->u3.e1.PageLocation];
// Must not remove pages from free or zeroed without updating
// the colored lists.
ASSERT (ListHead->ListName >= StandbyPageList);
// On MIPS R4000 modified pages destined for the paging file are
// kept on separate lists which group pages of the same color
// together
if ((ListHead == &MmModifiedPageListHead) &&
(Pfn->OriginalPte.u.Soft.Prototype == 0)) {
// This page is destined for the paging file (not
// a mapped file). Change the list head to the
// appropriate colored list head.
ListHead->Total -= 1;
MmTotalPagesForPagingFile -= 1;
ListHead = &MmModifiedPageListByColor [Pfn->u3.e1.PageColor];
}
ASSERT (Pfn->u3.e1.WriteInProgress == 0);
ASSERT (Pfn->u3.e1.ReadInProgress == 0);
ASSERT (ListHead->Total != 0);
Next = Pfn->u1.Flink;
Pfn->u1.Flink = 0; // Assumes Flink width is >= WsIndex width
Previous = Pfn->u2.Blink;
Pfn->u2.Blink = 0;
if (Next == MM_EMPTY_LIST) {
ListHead->Blink = Previous;
} else {
Pfn2 = MI_PFN_ELEMENT(Next);
Pfn2->u2.Blink = Previous;
}
if (Previous == MM_EMPTY_LIST) {
ListHead->Flink = Next;
} else {
Pfn2 = MI_PFN_ELEMENT(Previous);
Pfn2->u1.Flink = Next;
}
ListHead->Total -= 1;
// Check to see if we now have one less page available.
if (ListHead->ListName <= StandbyPageList) {
MmAvailablePages -= 1;
if (ListHead->ListName == StandbyPageList) {
MI_TALLY_TRANSITION_PAGE_REMOVAL (Pfn);
}
if (MmAvailablePages < MmMinimumFreePages) {
// Obtain free pages.
MiObtainFreePages();
}
}
else if (ListHead->ListName == ModifiedPageList || ListHead->ListName == ModifiedNoWritePageList) {
MI_TALLY_TRANSITION_PAGE_REMOVAL (Pfn);
}
return;
}
VOID
MiUnlinkFreeOrZeroedPage (
IN PFN_NUMBER Page
)
/*++
Routine Description:
This procedure removes a page from the middle of a list. This is
designed for the removing of free or zeroed pages from the middle of
their lists.
Arguments:
Pfn - Supplies a pointer to the PFN database element for the physical
page to remove from the list.
Return Value:
None.
Environment:
Must be holding the PFN database mutex with APCs disabled.
--*/
{
PMMPFNLIST ListHead;
PFN_NUMBER Previous;
PFN_NUMBER Next;
PMMPFN Pfn2;
PMMPFN Pfn;
ULONG Color;
Pfn = MI_PFN_ELEMENT (Page);
MM_PFN_LOCK_ASSERT();
ListHead = MmPageLocationList[Pfn->u3.e1.PageLocation];
ASSERT (ListHead->Total != 0);
ListHead->Total -= 1;
ASSERT (ListHead->ListName <= FreePageList);
ASSERT (Pfn->u3.e1.WriteInProgress == 0);
ASSERT (Pfn->u3.e1.ReadInProgress == 0);
PERFINFO_UNLINKFREEPAGE((ULONG_PTR)(Pfn - MmPfnDatabase), Pfn->u3.e1.PageLocation);
Next = Pfn->u1.Flink;
Pfn->u1.Flink = 0; // Assumes Flink width is >= WsIndex width
Previous = Pfn->u2.Blink;
Pfn->u2.Blink = 0;
if (Next == MM_EMPTY_LIST) {
ListHead->Blink = Previous;
} else {
Pfn2 = MI_PFN_ELEMENT(Next);
Pfn2->u2.Blink = Previous;
}
if (Previous == MM_EMPTY_LIST) {
ListHead->Flink = Next;
} else {
Pfn2 = MI_PFN_ELEMENT(Previous);
Pfn2->u1.Flink = Next;
}
// We are removing a page from the middle of the free or zeroed page list.
// The secondary color tables must be updated at this time.
Color = MI_GET_SECONDARY_COLOR (Page, Pfn);
ASSERT (Pfn->u3.e1.PageColor == MI_GET_COLOR_FROM_SECONDARY(Color));
// Walk down the list and remove the page.
Next = MmFreePagesByColor[ListHead->ListName][Color].Flink;
if (Next == Page) {
MmFreePagesByColor[ListHead->ListName][Color].Flink =
(PFN_NUMBER) Pfn->OriginalPte.u.Long;
} else {
// Walk the list to find the parent.
for (; ; ) {
Pfn2 = MI_PFN_ELEMENT (Next);
Next = (PFN_NUMBER) Pfn2->OriginalPte.u.Long;
if (Page == Next) {
Pfn2->OriginalPte.u.Long = Pfn->OriginalPte.u.Long;
if ((PFN_NUMBER) Pfn->OriginalPte.u.Long == MM_EMPTY_LIST) {
MmFreePagesByColor[ListHead->ListName][Color].Blink = Pfn2;
}
break;
}
}
}
MmAvailablePages -= 1;
if (MmAvailablePages < MmMinimumFreePages) {
// Obtain free pages.
MiObtainFreePages();
}
return;
}
ULONG
FASTCALL
MiEnsureAvailablePageOrWait (
IN PEPROCESS Process,
IN PVOID VirtualAddress
)
/*++
Routine Description:
This procedure ensures that a physical page is available on
the zeroed, free or standby list such that the next call the remove a
page absolutely will not block. This is necessary as blocking would
require a wait which could cause a deadlock condition.
If a page is available the function returns immediately with a value
of FALSE indicating no wait operation was performed. If no physical
page is available, the thread enters a wait state and the function
returns the value TRUE when the wait operation completes.
Arguments:
Process - Supplies a pointer to the current process if, and only if,
the working set mutex is held currently held and should
be released if a wait operation is issued. Supplies
the value NULL otherwise.
VirtualAddress - Supplies the virtual address for the faulting page.
If the value is NULL, the page is treated as a
user mode address.
Return Value:
FALSE - if a page was immediately available.
TRUE - if a wait operation occurred before a page became available.
Environment:
Must be holding the PFN database mutex with APCs disabled.
--*/
{
PVOID Event;
NTSTATUS Status;
KIRQL OldIrql;
KIRQL Ignore;
ULONG Limit;
ULONG Relock;
PFN_NUMBER StrandedPages;
LOGICAL WsHeldSafe;
PMMPFN Pfn1;
PMMPFN EndPfn;
LARGE_INTEGER WaitBegin;
LARGE_INTEGER WaitEnd;
MM_PFN_LOCK_ASSERT();
if (MmAvailablePages >= MM_HIGH_LIMIT) {
// Pages are available.
return FALSE;
}
// If this fault is for paged pool (or pagable kernel space,
// including page table pages), let it use the last page.
#if defined(_IA64_)
if (MI_IS_SYSTEM_ADDRESS(VirtualAddress) ||
(MI_IS_HYPER_SPACE_ADDRESS(VirtualAddress))) {
#else
if (((PMMPTE)VirtualAddress > MiGetPteAddress(HYPER_SPACE)) ||
((VirtualAddress > MM_HIGHEST_USER_ADDRESS) &&
(VirtualAddress < (PVOID)PTE_BASE))) {
#endif
// This fault is in the system, use 1 page as the limit.
if (MmAvailablePages >= MM_LOW_LIMIT) {
// Pages are available.
return FALSE;
}
Limit = MM_LOW_LIMIT;
Event = (PVOID)&MmAvailablePagesEvent;
} else {
Limit = MM_HIGH_LIMIT;
Event = (PVOID)&MmAvailablePagesEventHigh;
}
while (MmAvailablePages < Limit) {
KeClearEvent ((PKEVENT)Event);
UNLOCK_PFN (APC_LEVEL);
if (Process == HYDRA_PROCESS) {
UNLOCK_SESSION_SPACE_WS (APC_LEVEL);
}
else if (Process != NULL) {
// The working set lock may have been acquired safely or unsafely
// by our caller. Handle both cases here and below.
UNLOCK_WS_REGARDLESS (Process, WsHeldSafe);
}
else {
Relock = FALSE;
if (MmSystemLockOwner == PsGetCurrentThread()) {
UNLOCK_SYSTEM_WS (APC_LEVEL);
Relock = TRUE;
}
}
KiQueryInterruptTime(&WaitBegin);
// Wait 7 minutes for pages to become available.
Status = KeWaitForSingleObject(Event,
WrFreePage,
KernelMode,
FALSE,
(PLARGE_INTEGER)&MmSevenMinutes);
if (Status == STATUS_TIMEOUT) {
KiQueryInterruptTime(&WaitEnd);
// See how many transition pages have nonzero reference counts as
// these indicate drivers that aren't unlocking the pages in their
// MDLs.
Limit = 0;
StrandedPages = 0;
do {
Pfn1 = MI_PFN_ELEMENT (MmPhysicalMemoryBlock->Run[Limit].BasePage);
EndPfn = Pfn1 + MmPhysicalMemoryBlock->Run[Limit].PageCount;
while (Pfn1 < EndPfn) {
if ((Pfn1->u3.e1.PageLocation == TransitionPage) &&
(Pfn1->u3.e2.ReferenceCount != 0)) {
StrandedPages += 1;
}
Pfn1 += 1;
}
Limit += 1;
} while (Limit != MmPhysicalMemoryBlock->NumberOfRuns);
// This bugcheck can occur for the following reasons:
// A driver has blocked, deadlocking the modified or mapped
// page writers. Examples of this include mutex deadlocks or
// accesses to paged out memory in filesystem drivers, filter
// drivers, etc. This indicates a driver bug.
// The storage driver(s) are not processing requests. Examples
// of this are stranded queues, non-responding drives, etc. This
// indicates a driver bug.
// Not enough pool is available for the storage stack to write out
// modified pages. This indicates a driver bug.
// A high priority realtime thread has starved the balance set
// manager from trimming pages and/or starved the modified writer
// from writing them out. This indicates a bug in the component
// that created this thread.
// All the processes have been trimmed to their minimums and all
// modified pages written, but still no memory is available. The
// freed memory must be stuck in transition pages with non-zero
// reference counts - thus they cannot be put on the freelist.
// A driver is neglecting to unlock the pages preventing the
// reference counts from going to zero which would free the pages.
// This may be due to transfers that never finish and the driver
// never aborts or other driver bugs.
KeBugCheckEx (NO_PAGES_AVAILABLE,
MmModifiedPageListHead.Total,
MmTotalPagesForPagingFile,
(MmMaximumNonPagedPoolInBytes >> PAGE_SHIFT) - MmAllocatedNonPagedPool,
StrandedPages);
if (!KdDebuggerNotPresent) {
DbgPrint ("MmEnsureAvailablePageOrWait: 7 min timeout %x %x %x %x\n", WaitEnd.HighPart, WaitEnd.LowPart, WaitBegin.HighPart, WaitBegin.LowPart);
DbgBreakPoint ();
}
}
if (Process == HYDRA_PROCESS) {
LOCK_SESSION_SPACE_WS (Ignore);
}
else if (Process != NULL) {
// The working set lock may have been acquired safely or unsafely
// by our caller. Reacquire it in the same manner our caller did.
LOCK_WS_REGARDLESS (Process, WsHeldSafe);
}
else {
if (Relock) {
LOCK_SYSTEM_WS (Ignore);
}
}
LOCK_PFN (OldIrql);
}
return TRUE;
}
PFN_NUMBER //PageFrameIndex
FASTCALL
MiRemoveZeroPage (
IN ULONG PageColor
)
/*++
Routine Description:
This procedure removes a zero page from either the zeroed, free
or standby lists (in that order). If no pages exist on the zeroed
or free list a transition page is removed from the standby list
and the PTE (may be a prototype PTE) which refers to this page is
changed from transition back to its original contents.
If the page is not obtained from the zeroed list, it is zeroed.
Arguments:
PageColor - Supplies the page color for which this page is destined.
This is used for checking virtual address alignments to
determine if the D cache needs flushing before the page
can be reused.
Return Value:
The physical page number removed from the specified list.
Environment:
Must be holding the PFN database mutex with APCs disabled.
--*/
{
PFN_NUMBER Page;
PMMPFN Pfn1;
ULONG Color;
MM_PFN_LOCK_ASSERT();
ASSERT(MmAvailablePages != 0);
// Attempt to remove a page from the zeroed page list. If a page
// is available, then remove it and return its page frame index.
// Otherwise, attempt to remove a page from the free page list or
// the standby list.
// N.B. It is not necessary to change page colors even if the old
// color is not equal to the new color. The zero page thread
// ensures that all zeroed pages are removed from all caches.
if (MmFreePagesByColor[ZeroedPageList][PageColor].Flink != MM_EMPTY_LIST) {
// Remove the first entry on the zeroed by color list.
Page = MmFreePagesByColor[ZeroedPageList][PageColor].Flink;
#if DBG
Pfn1 = MI_PFN_ELEMENT(Page);
ASSERT (Pfn1->u3.e1.PageLocation == ZeroedPageList);
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
#endif
MiRemovePageByColor (Page, PageColor);
#if DBG
ASSERT (Pfn1->u3.e1.PageColor == (PageColor & MM_COLOR_MASK));
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
ASSERT (Pfn1->u2.ShareCount == 0);
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
#endif
return Page;
}
// No previously zeroed page with the specified secondary color exists.
// Try a zeroed page of the primary color.
if (MmZeroedPageListHead.Flink != MM_EMPTY_LIST) {
Page = MmZeroedPageListHead.Flink;
#if DBG
Pfn1 = MI_PFN_ELEMENT(Page);
ASSERT (Pfn1->u3.e1.PageLocation == ZeroedPageList);
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
#endif
Color = MI_GET_SECONDARY_COLOR (Page, MI_PFN_ELEMENT(Page));
MiRemovePageByColor (Page, Color);
#if DBG
Pfn1 = MI_PFN_ELEMENT(Page);
ASSERT (Pfn1->u3.e1.PageColor == (PageColor & MM_COLOR_MASK));
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
ASSERT (Pfn1->u2.ShareCount == 0);
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
#endif
return Page;
}
// No zeroed page of the primary color exists, try a free page of the
// secondary color.
if (MmFreePagesByColor[FreePageList][PageColor].Flink != MM_EMPTY_LIST) {
// Remove the first entry on the free list by color.
Page = MmFreePagesByColor[FreePageList][PageColor].Flink;
#if DBG
Pfn1 = MI_PFN_ELEMENT(Page);
ASSERT (Pfn1->u3.e1.PageLocation == FreePageList);
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
#endif
MiRemovePageByColor (Page, PageColor);
#if DBG
Pfn1 = MI_PFN_ELEMENT(Page);
ASSERT (Pfn1->u3.e1.PageColor == (PageColor & MM_COLOR_MASK));
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
ASSERT (Pfn1->u2.ShareCount == 0);
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
#endif
goto ZeroPage;
}
if (MmFreePageListHead.Flink != MM_EMPTY_LIST) {
Page = MmFreePageListHead.Flink;
Color = MI_GET_SECONDARY_COLOR (Page, MI_PFN_ELEMENT(Page));
MiRemovePageByColor (Page, Color);
#if DBG
Pfn1 = MI_PFN_ELEMENT(Page);
ASSERT (Pfn1->u3.e1.PageColor == (PageColor & MM_COLOR_MASK));
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
ASSERT (Pfn1->u2.ShareCount == 0);
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
#endif
goto ZeroPage;
}
ASSERT (MmZeroedPageListHead.Total == 0);
ASSERT (MmFreePageListHead.Total == 0);
if (MmZeroedPageListHead.Total != 0) {
Page = MiRemovePageFromList(&MmZeroedPageListHead);
MI_CHECK_PAGE_ALIGNMENT(Page, PageColor & MM_COLOR_MASK);
} else {
// Attempt to remove a page from the free list. If a page is
// available, then remove it. Otherwise, attempt to remove a
// page from the standby list.
if (MmFreePageListHead.Total != 0) {
Page = MiRemovePageFromList(&MmFreePageListHead);
ASSERT ((MI_PFN_ELEMENT(Page))->PteFrame != MI_MAGIC_AWE_PTEFRAME);
} else {
// Remove a page from the standby list and restore the original
// contents of the PTE to free the last reference to the physical
// page.
ASSERT (MmStandbyPageListHead.Total != 0);
Page = MiRemovePageFromList(&MmStandbyPageListHead);
ASSERT ((MI_PFN_ELEMENT(Page))->PteFrame != MI_MAGIC_AWE_PTEFRAME);
}
// Zero the page removed from the free or standby list.
ZeroPage:
Pfn1 = MI_PFN_ELEMENT(Page);
#if defined(_ALPHA_)
HalZeroPage((PVOID)ULongToPtr((PageColor & MM_COLOR_MASK) << PAGE_SHIFT),
(PVOID)ULongToPtr((Pfn1->u3.e1.PageColor) << PAGE_SHIFT),
Page);
#else
MiZeroPhysicalPage (Page, 0);
#endif
// Note the stamping must occur after the page is zeroed.
MI_BARRIER_STAMP_ZEROED_PAGE (&Pfn1->PteFrame);
Pfn1->u3.e1.PageColor = PageColor & MM_COLOR_MASK;
}
#if DBG
Pfn1 = MI_PFN_ELEMENT (Page);
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
ASSERT (Pfn1->u2.ShareCount == 0);
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
#endif
return Page;
}
PFN_NUMBER //PageFrameIndex
FASTCALL
MiRemoveAnyPage (
IN ULONG PageColor
)
/*++
Routine Description:
This procedure removes a page from either the free, zeroed,
or standby lists (in that order). If no pages exist on the zeroed
or free list a transition page is removed from the standby list
and the PTE (may be a prototype PTE) which refers to this page is
changed from transition back to its original contents.
Note pages MUST exist to satisfy this request. The caller ensures this
by first calling MiEnsureAvailablePageOrWait.
Arguments:
PageColor - Supplies the page color for which this page is destined.
This is used for checking virtual address alignments to
determine if the D cache needs flushing before the page
can be reused.
Return Value:
The physical page number removed from the specified list.
Environment:
Must be holding the PFN database mutex with APCs disabled.
--*/
{
PFN_NUMBER Page;
PMMPFN Pfn1;
ULONG Color;
MM_PFN_LOCK_ASSERT();
ASSERT(MmAvailablePages != 0);
// Check the free page list, and if a page is available
// remove it and return its value.
if (MmFreePagesByColor[FreePageList][PageColor].Flink != MM_EMPTY_LIST) {
// Remove the first entry on the free by color list.
Page = MmFreePagesByColor[FreePageList][PageColor].Flink;
#if DBG
Pfn1 = MI_PFN_ELEMENT(Page);
ASSERT (Pfn1->u3.e1.PageLocation == FreePageList);
ASSERT (Pfn1->u3.e1.PageColor == (PageColor & MM_COLOR_MASK));
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
#endif
MiRemovePageByColor (Page, PageColor);
#if DBG
ASSERT (Pfn1->u3.e1.PageColor == (PageColor & MM_COLOR_MASK));
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
ASSERT (Pfn1->u2.ShareCount == 0);
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
#endif
return Page;
}
if (MmFreePagesByColor[ZeroedPageList][PageColor].Flink
!= MM_EMPTY_LIST) {
// Remove the first entry on the zeroed by color list.
Page = MmFreePagesByColor[ZeroedPageList][PageColor].Flink;
#if DBG
Pfn1 = MI_PFN_ELEMENT(Page);
ASSERT (Pfn1->u3.e1.PageColor == (PageColor & MM_COLOR_MASK));
ASSERT (Pfn1->u3.e1.PageLocation == ZeroedPageList);
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
#endif
MiRemovePageByColor (Page, PageColor);
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
return Page;
}
// Try the free page list by primary color.
if (MmFreePageListHead.Flink != MM_EMPTY_LIST) {
Page = MmFreePageListHead.Flink;
Color = MI_GET_SECONDARY_COLOR (Page, MI_PFN_ELEMENT(Page));
#if DBG
Pfn1 = MI_PFN_ELEMENT(Page);
ASSERT (Pfn1->u3.e1.PageColor == (PageColor & MM_COLOR_MASK));
ASSERT (Pfn1->u3.e1.PageLocation == FreePageList);
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
#endif
MiRemovePageByColor (Page, Color);
#if DBG
ASSERT (Pfn1->u3.e1.PageColor == (PageColor & MM_COLOR_MASK));
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
ASSERT (Pfn1->u2.ShareCount == 0);
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
#endif
return Page;
}
if (MmZeroedPageListHead.Flink != MM_EMPTY_LIST) {
Page = MmZeroedPageListHead.Flink;
Color = MI_GET_SECONDARY_COLOR (Page, MI_PFN_ELEMENT(Page));
MiRemovePageByColor (Page, Color);
#if DBG
Pfn1 = MI_PFN_ELEMENT(Page);
ASSERT (Pfn1->u3.e1.PageColor == (PageColor & MM_COLOR_MASK));
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
ASSERT (Pfn1->u2.ShareCount == 0);
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
#endif
return Page;
}
if (MmFreePageListHead.Total != 0) {
Page = MiRemovePageFromList(&MmFreePageListHead);
ASSERT ((MI_PFN_ELEMENT(Page))->PteFrame != MI_MAGIC_AWE_PTEFRAME);
} else {
// Check the zeroed page list, and if a page is available
// remove it and return its value.
if (MmZeroedPageListHead.Total != 0) {
Page = MiRemovePageFromList(&MmZeroedPageListHead);
ASSERT ((MI_PFN_ELEMENT(Page))->PteFrame != MI_MAGIC_AWE_PTEFRAME);
} else {
// No pages exist on the free or zeroed list, use the
// standby list.
ASSERT(MmStandbyPageListHead.Total != 0);
Page = MiRemovePageFromList(&MmStandbyPageListHead);
ASSERT ((MI_PFN_ELEMENT(Page))->PteFrame != MI_MAGIC_AWE_PTEFRAME);
}
}
MI_CHECK_PAGE_ALIGNMENT(Page, PageColor & MM_COLOR_MASK);
#if DBG
Pfn1 = MI_PFN_ELEMENT (Page);
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
ASSERT (Pfn1->u2.ShareCount == 0);
#endif
return Page;
}
VOID
MiRemovePageByColor (
IN PFN_NUMBER Page,
IN ULONG Color
)
/*++
Routine Description:
This procedure removes a page from the middle of the free or
zeroed page list.
Arguments:
PageFrameIndex - Supplies the physical page number to unlink from the
list.
Return Value:
none.
Environment:
Must be holding the PFN database mutex with APCs disabled.
--*/
{
PMMPFNLIST ListHead;
PMMPFNLIST PrimaryListHead;
PFN_NUMBER Previous;
PFN_NUMBER Next;
PMMPFN Pfn1;
PMMPFN Pfn2;
ULONG PrimaryColor;
MM_PFN_LOCK_ASSERT();
Pfn1 = MI_PFN_ELEMENT (Page);
PrimaryColor = Pfn1->u3.e1.PageColor;
PERFINFO_REMOVEPAGE(Page, PERFINFO_LOG_TYPE_REMOVEPAGEBYCOLOR);
ListHead = MmPageLocationList[Pfn1->u3.e1.PageLocation];
ListHead->Total -= 1;
PrimaryListHead = ListHead;
#if PFN_CONSISTENCY
if (MmFreePagesByColor[PrimaryListHead->ListName][Color].Flink != Page) {
KeBugCheckEx (PFN_LIST_CORRUPT,
0x9A,
Page,
Color,
(PFN_NUMBER)&MmFreePagesByColor[PrimaryListHead->ListName][Color].Flink);
}
#endif
Next = Pfn1->u1.Flink;
Pfn1->u1.Flink = 0; // Assumes Flink width is >= WsIndex width
Previous = Pfn1->u2.Blink;
Pfn1->u2.Blink = 0;
if (Next == MM_EMPTY_LIST) {
PrimaryListHead->Blink = Previous;
} else {
Pfn2 = MI_PFN_ELEMENT(Next);
Pfn2->u2.Blink = Previous;
}
if (Previous == MM_EMPTY_LIST) {
PrimaryListHead->Flink = Next;
} else {
Pfn2 = MI_PFN_ELEMENT(Previous);
Pfn2->u1.Flink = Next;
}
// Zero the flags longword, but keep the color information.
ASSERT (Pfn1->u3.e1.RemovalRequested == 0);
Pfn1->u3.e2.ShortFlags = 0;
Pfn1->u3.e1.PageColor = PrimaryColor;
// Update the color lists.
MmFreePagesByColor[ListHead->ListName][Color].Flink =
(PFN_NUMBER) Pfn1->OriginalPte.u.Long;
// Note that we now have one less page available.
MmAvailablePages -= 1;
if (MmAvailablePages < MmMinimumFreePages) {
// Obtain free pages.
MiObtainFreePages();
}
return;
}
VOID
FASTCALL
MiInsertFrontModifiedNoWrite (
IN PFN_NUMBER PageFrameIndex
)
/*++
Routine Description:
This procedure inserts a page at the FRONT of the modified no
write list.
Arguments:
PageFrameIndex - Supplies the physical page number to insert in the
list.
Return Value:
none.
Environment:
Must be holding the PFN database mutex with APCs disabled.
--*/
{
PFN_NUMBER first;
PMMPFN Pfn1;
PMMPFN Pfn2;
MM_PFN_LOCK_ASSERT();
ASSERT ((PageFrameIndex != 0) && (PageFrameIndex <= MmHighestPhysicalPage) &&
(PageFrameIndex >= MmLowestPhysicalPage));
// Check to ensure the reference count for the page
// is zero.
Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
MmModifiedNoWritePageListHead.Total += 1; // One more page on the list.
MI_TALLY_TRANSITION_PAGE_ADDITION (Pfn1);
first = MmModifiedNoWritePageListHead.Flink;
if (first == MM_EMPTY_LIST) {
// List is empty add the page to the ListHead.
MmModifiedNoWritePageListHead.Blink = PageFrameIndex;
} else {
Pfn2 = MI_PFN_ELEMENT (first);
Pfn2->u2.Blink = PageFrameIndex;
}
MmModifiedNoWritePageListHead.Flink = PageFrameIndex;
Pfn1->u1.Flink = first;
Pfn1->u2.Blink = MM_EMPTY_LIST;
Pfn1->u3.e1.PageLocation = ModifiedNoWritePageList;
return;
}
Reference in New Issue Copy Permalink