1792 lines
48 KiB
C
1792 lines
48 KiB
C
/*++
|
|
|
|
Copyright (c) 1989-2002 Microsoft Corporation
|
|
|
|
Module Name:
|
|
|
|
pfnsup.c
|
|
|
|
Abstract:
|
|
|
|
This module implements routines which provide support for managing the page
|
|
frame database.
|
|
|
|
--*/
|
|
|
|
#include "mi.h"
|
|
|
|
//
|
|
// Define a macro to statically initialize a PFN region.
|
|
//
|
|
#define MI_PFN_REGION_INITIALIZER() { \
|
|
{ \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
{ MM_PACKED_PFN_NULL, MM_PACKED_PFN_NULL }, \
|
|
}, \
|
|
0, \
|
|
}
|
|
|
|
//
|
|
// Total number of physical pages available on the system.
|
|
//
|
|
DECLSPEC_STICKY PFN_COUNT MmNumberOfPhysicalPages;
|
|
|
|
//
|
|
// Highest physical page number in the system.
|
|
//
|
|
#ifdef CONSOLE_DEVKIT
|
|
PFN_NUMBER MmHighestPhysicalPage;
|
|
#else
|
|
DECLSPEC_RDATA PFN_NUMBER MmHighestPhysicalPage = MM_HIGHEST_PHYSICAL_PAGE;
|
|
#endif
|
|
|
|
//
|
|
// Database that contains the use of each allocatable physical page in the
|
|
// system. Exposed here for the kernel debugger.
|
|
//
|
|
DECLSPEC_SELECTANY PMMPFN MmPfnDatabase = MM_PFN_DATABASE;
|
|
|
|
//
|
|
// Region for general use pages, such as code, pool, and stacks.
|
|
//
|
|
MMPFNREGION MmRetailPfnRegion = MI_PFN_REGION_INITIALIZER();
|
|
|
|
#ifdef CONSOLE_DEVKIT
|
|
//
|
|
// Region for the extra memory available in the development kits.
|
|
//
|
|
MMPFNREGION MmDeveloperKitPfnRegion = MI_PFN_REGION_INITIALIZER();
|
|
|
|
//
|
|
// Mapping from MI_PFN_REGION to the corresponding PMMPFNREGION.
|
|
//
|
|
PMMPFNREGION MmPfnRegions[MI_NUMBER_OF_REGIONS] = {
|
|
&MmRetailPfnRegion,
|
|
&MmDeveloperKitPfnRegion,
|
|
&MmDeveloperKitPfnRegion,
|
|
&MmDeveloperKitPfnRegion,
|
|
};
|
|
|
|
//
|
|
// Set if MmReleaseDeveloperKitMemory has been called to place all of the extra
|
|
// pages available on a developer kit in the general free lists.
|
|
//
|
|
BOOLEAN MmDeveloperKitMemoryReleased;
|
|
#endif
|
|
|
|
//
|
|
// Page color to use for the next allocation without a specified page color.
|
|
//
|
|
MMCOLOR MmNextDefaultPageColor;
|
|
|
|
//
|
|
// Number of pages that can be allocated for general use.
|
|
//
|
|
PFN_COUNT MmAvailablePages;
|
|
|
|
//
|
|
// Number of pages that have been allocated for the various usage types.
|
|
//
|
|
PFN_COUNT MmAllocatedPagesByUsage[MmMaximumUsage];
|
|
|
|
//
|
|
// Local support.
|
|
//
|
|
|
|
VOID
|
|
MiRelocateBusyPageHelper(
|
|
IN PVOID OldPhysicalAddress,
|
|
IN PVOID NewPhysicalAddress,
|
|
IN PMMPTE PointerPte,
|
|
IN MMPTE NewPteContents,
|
|
IN BOOLEAN FullTlbFlush
|
|
);
|
|
|
|
#if DBG
|
|
VOID
|
|
MiAssertFreeListsValid(
|
|
VOID
|
|
);
|
|
#define MI_ASSERT_FREE_LISTS_VALID() MiAssertFreeListsValid()
|
|
#else
|
|
#define MI_ASSERT_FREE_LISTS_VALID()
|
|
#endif
|
|
|
|
#ifdef ALLOC_PRAGMA
|
|
#pragma alloc_text(INIT, MiInitializePfnDatabase)
|
|
#endif
|
|
|
|
VOID
|
|
MiInitializePfnDatabase(
|
|
VOID
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine initializes the database used to track busy and free pages.
|
|
|
|
Arguments:
|
|
|
|
None.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
Environment:
|
|
|
|
Kernel mode.
|
|
|
|
--*/
|
|
{
|
|
KIRQL OldIrql;
|
|
PIMAGE_NT_HEADERS NtHeader;
|
|
ULONG NtosImageSize;
|
|
PFN_COUNT PfnDatabaseSizeInPages;
|
|
MMPFN TempPageFrame;
|
|
ULONG PageFrameNumber;
|
|
ULONG EndingPageFrameNumber;
|
|
#ifdef DEVKIT
|
|
PXDATA_SECTION_HEADER DataSectionHeader;
|
|
#endif
|
|
PIMAGE_SECTION_HEADER TextSectionHeader;
|
|
|
|
MI_LOCK_MM(&OldIrql);
|
|
|
|
//
|
|
// Verify that the highest physical page doesn't exceed the limits of the
|
|
// physical mapping window.
|
|
//
|
|
|
|
ASSERT(MM_HIGHEST_PHYSICAL_PAGE < MM_PAGES_IN_PHYSICAL_MAP);
|
|
|
|
//
|
|
// Find the NT headers for XBOXKRNL.EXE.
|
|
//
|
|
|
|
NtHeader = RtlImageNtHeader(PsNtosImageBase);
|
|
|
|
ASSERT(NtHeader != NULL);
|
|
ASSERT(NtHeader->FileHeader.NumberOfSections > 0);
|
|
|
|
//
|
|
// Get the size of the XBOXKRNL.EXE image.
|
|
//
|
|
|
|
NtosImageSize = ROUND_TO_PAGES(NtHeader->OptionalHeader.SizeOfImage);
|
|
|
|
//
|
|
// Compute how many pages will be required to hold the PFN database.
|
|
//
|
|
|
|
PfnDatabaseSizeInPages = BYTES_TO_PAGES((MM_HIGHEST_PHYSICAL_PAGE + 1) *
|
|
sizeof(MMPFN));
|
|
|
|
//
|
|
// Initialize all entries in the PFN database as unknown usage with a
|
|
// maximum reference count so that MmAllocateContiguousMemoryEx doesn't
|
|
// consider the page when finding a candidate range.
|
|
//
|
|
|
|
TempPageFrame.Long = 0;
|
|
TempPageFrame.Busy.Busy = 1;
|
|
TempPageFrame.Busy.LockCount = MI_LOCK_COUNT_MAXIMUM;
|
|
TempPageFrame.Busy.BusyType = MmUnknownUsage;
|
|
|
|
RtlFillMemoryUlong(MM_PFN_DATABASE, PfnDatabaseSizeInPages * PAGE_SIZE,
|
|
TempPageFrame.Long);
|
|
|
|
//
|
|
// Insert all of the pages from the bottom of memory to the base of
|
|
// XBOXKRNL.EXE into the free lists.
|
|
//
|
|
|
|
MiInsertPhysicalMemoryInFreeList(MM_LOWEST_PHYSICAL_PAGE,
|
|
MI_CONVERT_PHYSICAL_TO_PFN(PsNtosImageBase));
|
|
|
|
#ifdef ARCADE
|
|
//
|
|
// Insert all of the pages from the end of XBOXKRNL.EXE to the base of the
|
|
// PFN database into the free lists.
|
|
//
|
|
|
|
MiInsertPhysicalMemoryInFreeList(
|
|
MI_CONVERT_PHYSICAL_TO_PFN((PUCHAR)PsNtosImageBase + NtosImageSize),
|
|
MM_DATABASE_PHYSICAL_PAGE);
|
|
#else
|
|
//
|
|
// Insert all of the pages from the end of XBOXKRNL.EXE to the base of the
|
|
// NVIDIA NV2A instance memory into the free lists.
|
|
//
|
|
|
|
MiInsertPhysicalMemoryInFreeList(
|
|
MI_CONVERT_PHYSICAL_TO_PFN((PUCHAR)PsNtosImageBase + NtosImageSize),
|
|
MM_INSTANCE_PHYSICAL_PAGE);
|
|
#endif
|
|
|
|
#ifdef CONSOLE_DEVKIT
|
|
//
|
|
// Insert all of the pages from the end of the PFN database to the top of
|
|
// memory into the free lists.
|
|
//
|
|
|
|
MiInsertPhysicalMemoryInFreeList(MM_DATABASE_PHYSICAL_PAGE +
|
|
PfnDatabaseSizeInPages, MM_HIGHEST_PHYSICAL_PAGE + 1);
|
|
#endif
|
|
|
|
#ifdef DEVKIT
|
|
//
|
|
// If we booted from a shadow copy of the ROM, then we need to remove the
|
|
// pages that were used to make a copy of the kernel's initialized data.
|
|
//
|
|
|
|
if (XboxBootFlags & XBOX_BOOTFLAG_SHADOW) {
|
|
|
|
DataSectionHeader = (PXDATA_SECTION_HEADER)((PIMAGE_DOS_HEADER)PsNtosImageBase)->e_res2;
|
|
|
|
PageFrameNumber = MI_CONVERT_PHYSICAL_TO_PFN(DataSectionHeader->PointerToRawData);
|
|
EndingPageFrameNumber = PageFrameNumber +
|
|
BYTES_TO_PAGES(DataSectionHeader->SizeOfInitializedData);
|
|
|
|
while (PageFrameNumber < EndingPageFrameNumber) {
|
|
|
|
MiRemovePageFromFreeList(PageFrameNumber);
|
|
|
|
TempPageFrame.Pte.Long = MiGetValidKernelPteBits();
|
|
TempPageFrame.Pte.Hard.Write = 0;
|
|
TempPageFrame.Pte.Hard.PersistAllocation = 1;
|
|
TempPageFrame.Pte.Hard.PageFrameNumber = PageFrameNumber;
|
|
|
|
*MI_PFN_ELEMENT(PageFrameNumber) = TempPageFrame;
|
|
|
|
MmAllocatedPagesByUsage[MmContiguousUsage]++;
|
|
|
|
PageFrameNumber++;
|
|
}
|
|
}
|
|
#endif
|
|
|
|
//
|
|
// Convert the page frame that holds the page directory to a page table
|
|
// entry.
|
|
//
|
|
|
|
PageFrameNumber = MI_CONVERT_PHYSICAL_TO_PFN(PAGE_DIRECTORY_PHYSICAL_ADDRESS);
|
|
|
|
MiRemovePageFromFreeList(PageFrameNumber);
|
|
|
|
TempPageFrame.Pte.Long = MiGetValidKernelPteBits();
|
|
TempPageFrame.Pte.Hard.PersistAllocation = 1;
|
|
TempPageFrame.Pte.Hard.GuardOrEndOfAllocation = 1;
|
|
TempPageFrame.Pte.Hard.PageFrameNumber = PageFrameNumber;;
|
|
|
|
*MI_PFN_ELEMENT(PageFrameNumber) = TempPageFrame;
|
|
|
|
MmAllocatedPagesByUsage[MmContiguousUsage]++;
|
|
|
|
//
|
|
// Convert the page frames that hold XBOXKRNL.EXE to page table entries.
|
|
//
|
|
|
|
PageFrameNumber = MI_CONVERT_PHYSICAL_TO_PFN(PsNtosImageBase);
|
|
EndingPageFrameNumber = MI_CONVERT_PHYSICAL_TO_PFN((PUCHAR)PsNtosImageBase +
|
|
NtosImageSize);
|
|
|
|
while (PageFrameNumber < EndingPageFrameNumber) {
|
|
|
|
TempPageFrame.Pte.Long = MiGetValidKernelPteBits();
|
|
TempPageFrame.Pte.Hard.PersistAllocation = 1;
|
|
TempPageFrame.Pte.Hard.PageFrameNumber = PageFrameNumber;
|
|
|
|
*MI_PFN_ELEMENT(PageFrameNumber) = TempPageFrame;
|
|
|
|
MmNumberOfPhysicalPages++;
|
|
MmAllocatedPagesByUsage[MmContiguousUsage]++;
|
|
|
|
PageFrameNumber++;
|
|
}
|
|
|
|
//
|
|
// Find the first section of XBOXKRNL.EXE, which by convention should be the
|
|
// text section.
|
|
//
|
|
|
|
TextSectionHeader = IMAGE_FIRST_SECTION(NtHeader);
|
|
|
|
ASSERT((*(PULONG)TextSectionHeader->Name) == 'xet.');
|
|
ASSERT((TextSectionHeader->Characteristics & IMAGE_SCN_CNT_CODE) != 0);
|
|
|
|
//
|
|
// Convert the image headers and .text section of XBOXKRNL.EXE to read-only.
|
|
//
|
|
|
|
PageFrameNumber = MI_CONVERT_PHYSICAL_TO_PFN(PsNtosImageBase);
|
|
EndingPageFrameNumber = MI_CONVERT_PHYSICAL_TO_PFN((PUCHAR)PsNtosImageBase +
|
|
TextSectionHeader->Misc.VirtualSize);
|
|
|
|
while (PageFrameNumber < EndingPageFrameNumber) {
|
|
MI_PFN_ELEMENT(PageFrameNumber)->Pte.Hard.Write = 0;
|
|
PageFrameNumber++;
|
|
}
|
|
|
|
//
|
|
// Convert the page frames that hold the PFN database to page table entries.
|
|
// The pages persist across a quick reboot.
|
|
//
|
|
|
|
PageFrameNumber = MM_DATABASE_PHYSICAL_PAGE;
|
|
EndingPageFrameNumber = MM_DATABASE_PHYSICAL_PAGE + PfnDatabaseSizeInPages;
|
|
|
|
while (PageFrameNumber < EndingPageFrameNumber) {
|
|
|
|
TempPageFrame.Pte.Long = MiGetValidKernelPteBits();
|
|
TempPageFrame.Pte.Hard.PersistAllocation = 1;
|
|
TempPageFrame.Pte.Hard.PageFrameNumber = PageFrameNumber;
|
|
|
|
*MI_PFN_ELEMENT(PageFrameNumber) = TempPageFrame;
|
|
|
|
MmNumberOfPhysicalPages++;
|
|
MmAllocatedPagesByUsage[MmContiguousUsage]++;
|
|
|
|
PageFrameNumber++;
|
|
}
|
|
|
|
//
|
|
// Convert the page frames that hold NV2A instance memory to page table
|
|
// entries.
|
|
//
|
|
|
|
PageFrameNumber = MM_INSTANCE_PHYSICAL_PAGE;
|
|
EndingPageFrameNumber = MM_INSTANCE_PHYSICAL_PAGE + MM_INSTANCE_PAGE_COUNT;
|
|
|
|
while (PageFrameNumber < EndingPageFrameNumber) {
|
|
|
|
TempPageFrame.Pte.Long = MiGetValidKernelPteBits();
|
|
MI_DISABLE_CACHING(TempPageFrame.Pte);
|
|
TempPageFrame.Pte.Hard.PageFrameNumber = PageFrameNumber;
|
|
|
|
*MI_PFN_ELEMENT(PageFrameNumber) = TempPageFrame;
|
|
|
|
MmNumberOfPhysicalPages++;
|
|
MmAllocatedPagesByUsage[MmContiguousUsage]++;
|
|
|
|
PageFrameNumber++;
|
|
}
|
|
|
|
#ifdef CONSOLE_DEVKIT
|
|
//
|
|
// On a development kit system, the NVIDIA NV2A instance memory must be
|
|
// relative to the top of memory, so we also reserve the instance memory
|
|
// pages in the upper half of memory.
|
|
//
|
|
|
|
if (MM_HIGHEST_PHYSICAL_PAGE != MM_64M_PHYSICAL_PAGE - 1) {
|
|
|
|
PageFrameNumber = MM_INSTANCE_PHYSICAL_PAGE + MM_64M_PHYSICAL_PAGE;
|
|
EndingPageFrameNumber = MM_INSTANCE_PHYSICAL_PAGE +
|
|
MM_INSTANCE_PAGE_COUNT + MM_64M_PHYSICAL_PAGE;
|
|
|
|
while (PageFrameNumber < EndingPageFrameNumber) {
|
|
|
|
MiRemovePageFromFreeList(PageFrameNumber);
|
|
|
|
TempPageFrame.Pte.Long = MiGetValidKernelPteBits();
|
|
MI_DISABLE_CACHING(TempPageFrame.Pte);
|
|
TempPageFrame.Pte.Hard.PageFrameNumber = PageFrameNumber;
|
|
|
|
*MI_PFN_ELEMENT(PageFrameNumber) = TempPageFrame;
|
|
|
|
MmAllocatedPagesByUsage[MmContiguousUsage]++;
|
|
|
|
PageFrameNumber++;
|
|
}
|
|
}
|
|
#endif
|
|
|
|
//
|
|
// Reserve the first physical page in the system for D3D's use.
|
|
//
|
|
|
|
MiRemovePageFromFreeList(MM_D3D_PHYSICAL_PAGE);
|
|
|
|
TempPageFrame.Pte.Long = MiGetValidKernelPteBits();
|
|
TempPageFrame.Pte.Hard.PersistAllocation = 1;
|
|
TempPageFrame.Pte.Hard.GuardOrEndOfAllocation = 1;
|
|
TempPageFrame.Pte.Hard.PageFrameNumber = MM_D3D_PHYSICAL_PAGE;
|
|
|
|
*MI_PFN_ELEMENT(MM_D3D_PHYSICAL_PAGE) = TempPageFrame;
|
|
|
|
MmAllocatedPagesByUsage[MmContiguousUsage]++;
|
|
|
|
MI_UNLOCK_MM(OldIrql);
|
|
}
|
|
|
|
VOID
|
|
MiReinitializePfnDatabase(
|
|
VOID
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine reinitializes the database used to track busy and free pages
|
|
after a quick reboot has occurred.
|
|
|
|
Arguments:
|
|
|
|
None.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
Environment:
|
|
|
|
Kernel mode.
|
|
|
|
--*/
|
|
{
|
|
KIRQL OldIrql;
|
|
PFN_NUMBER PageFrameNumber;
|
|
PMMPFN PageFrame;
|
|
|
|
MI_LOCK_MM(&OldIrql);
|
|
|
|
//
|
|
// Loop over the page frame database and either keep persistent pages or
|
|
// insert pages into the free lists.
|
|
//
|
|
|
|
for (PageFrameNumber = MM_LOWEST_PHYSICAL_PAGE; PageFrameNumber <=
|
|
MM_HIGHEST_PHYSICAL_PAGE; PageFrameNumber++) {
|
|
|
|
PageFrame = MI_PFN_ELEMENT(PageFrameNumber);
|
|
|
|
if (PageFrame->Pte.Hard.Valid != 0) {
|
|
|
|
//
|
|
// The page was a contiguous memory allocation in the previous
|
|
// instance of the kernel. If the page is marked as being
|
|
// peristent, then keep the allocation alive.
|
|
//
|
|
|
|
if (PageFrame->Pte.Hard.PersistAllocation != 0) {
|
|
MmAllocatedPagesByUsage[MmContiguousUsage]++;
|
|
continue;
|
|
}
|
|
|
|
} else if (PageFrame->Busy.Busy != 0) {
|
|
|
|
//
|
|
// The page was marked as an unknown page type in the previous
|
|
// instance of the kernel. Keep the page in this state since we
|
|
// can't manage the page.
|
|
//
|
|
|
|
if (PageFrame->Busy.BusyType == MmUnknownUsage) {
|
|
continue;
|
|
}
|
|
|
|
} else {
|
|
|
|
//
|
|
// The page was free in the previous instance of the kernel.
|
|
//
|
|
}
|
|
|
|
//
|
|
// MmAllocateContiguousMemorySpecifyCache searches for free pages from
|
|
// the highest page to the lowest page, so insert pages such that the
|
|
// lowest pages are at the front of the free lists. This avoids
|
|
// unnecessary page relocation in order to satisfy a contiguous memory
|
|
// allocation.
|
|
//
|
|
|
|
MiInsertPageInFreeList(PageFrameNumber, FALSE);
|
|
}
|
|
|
|
MI_UNLOCK_MM(OldIrql);
|
|
}
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiInsertPageInFreeList(
|
|
IN PFN_NUMBER PageFrameNumber,
|
|
IN BOOLEAN InsertAtHeadList
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine inserts the supplied physical page into the appropriate free
|
|
list. No assumptions are made about the current status of the page.
|
|
|
|
Arguments:
|
|
|
|
PageFrameNumber - Supplies the physical page frame to insert.
|
|
|
|
InsertAtHeadList - If TRUE, insert the page at the front of the free lists,
|
|
else insert the page at the end of the free lists.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
Environment:
|
|
|
|
Kernel mode, DISPATCH_LEVEL.
|
|
|
|
--*/
|
|
{
|
|
MMCOLOR Color;
|
|
USHORT PackedPfn;
|
|
PMMPFN PageFrame;
|
|
PMMPFNREGION PfnRegion;
|
|
PMMPFN OldLinkPageFrame;
|
|
|
|
MI_ASSERT_LOCK_MM();
|
|
|
|
ASSERT(PageFrameNumber <= MM_HIGHEST_PHYSICAL_PAGE);
|
|
|
|
Color = MiGetPfnColor(PageFrameNumber);
|
|
PackedPfn = MiPackFreePfn(PageFrameNumber);
|
|
|
|
//
|
|
// Verify that the caller is not attempting to free the page frame number
|
|
// used as the end of list marker.
|
|
//
|
|
|
|
ASSERT(PackedPfn != MM_PACKED_PFN_NULL);
|
|
|
|
//
|
|
// Copy the list head to the freed page frame. We'll clear out the forward
|
|
// or backward link below depending on the list insertion direction.
|
|
//
|
|
|
|
PageFrame = MI_PFN_ELEMENT(PageFrameNumber);
|
|
PfnRegion = MI_PFN_REGION(PageFrameNumber);
|
|
PageFrame->Free = PfnRegion->FreePagesByColor[Color];
|
|
|
|
if (PfnRegion->FreePagesByColor[Color].PackedPfnFlink != MM_PACKED_PFN_NULL) {
|
|
|
|
//
|
|
// The page color list was not empty, so link the new page list head or
|
|
// tail to the old page list head or tail.
|
|
//
|
|
|
|
if (InsertAtHeadList) {
|
|
|
|
PageFrame->Free.PackedPfnBlink = MM_PACKED_PFN_NULL;
|
|
PfnRegion->FreePagesByColor[Color].PackedPfnFlink = PackedPfn;
|
|
|
|
OldLinkPageFrame = MiUnpackFreePfnElement(PageFrame->Free.PackedPfnFlink,
|
|
Color);
|
|
|
|
ASSERT(OldLinkPageFrame->Free.PackedPfnBlink == MM_PACKED_PFN_NULL);
|
|
OldLinkPageFrame->Free.PackedPfnBlink = PackedPfn;
|
|
|
|
} else {
|
|
|
|
PageFrame->Free.PackedPfnFlink = MM_PACKED_PFN_NULL;
|
|
PfnRegion->FreePagesByColor[Color].PackedPfnBlink = PackedPfn;
|
|
|
|
OldLinkPageFrame = MiUnpackFreePfnElement(PageFrame->Free.PackedPfnBlink,
|
|
Color);
|
|
|
|
ASSERT(OldLinkPageFrame->Free.PackedPfnFlink == MM_PACKED_PFN_NULL);
|
|
OldLinkPageFrame->Free.PackedPfnFlink = PackedPfn;
|
|
}
|
|
|
|
ASSERT(OldLinkPageFrame->Busy.Busy == 0);
|
|
|
|
} else {
|
|
|
|
//
|
|
// The page color list was empty, so make both the head and tail
|
|
// pointer reference this page.
|
|
//
|
|
|
|
ASSERT(PageFrame->Free.PackedPfnFlink == MM_PACKED_PFN_NULL);
|
|
ASSERT(PageFrame->Free.PackedPfnBlink == MM_PACKED_PFN_NULL);
|
|
|
|
PfnRegion->FreePagesByColor[Color].PackedPfnBlink = PackedPfn;
|
|
PfnRegion->FreePagesByColor[Color].PackedPfnFlink = PackedPfn;
|
|
}
|
|
|
|
//
|
|
// Verify that the packed page frame numbers are structurally correct-- the
|
|
// low bits must be clear to mark the page as free.
|
|
//
|
|
|
|
ASSERT(PageFrame->Pte.Hard.Valid == 0);
|
|
ASSERT(PageFrame->Busy.Busy == 0);
|
|
|
|
//
|
|
// Update the number of available pages in the region.
|
|
//
|
|
|
|
PfnRegion->AvailablePages++;
|
|
|
|
#ifdef CONSOLE_DEVKIT
|
|
//
|
|
// Update the number of available pages in the system if the page is from
|
|
// the retail PFN region or if the extra memory on a development kit is
|
|
// available for general use.
|
|
//
|
|
|
|
if (PfnRegion == &MmRetailPfnRegion || MmDeveloperKitMemoryReleased) {
|
|
MmAvailablePages++;
|
|
}
|
|
#else
|
|
//
|
|
// Update the number of available pages in the system.
|
|
//
|
|
|
|
MmAvailablePages++;
|
|
#endif
|
|
}
|
|
|
|
VOID
|
|
MiInsertPhysicalMemoryInFreeList(
|
|
IN PFN_NUMBER PageFrameNumber,
|
|
IN PFN_NUMBER EndingPageFrameNumberExclusive
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine inserts a range of physical pages into the appropriate free
|
|
lists. No assumptions are made about the current status of the pages.
|
|
|
|
Arguments:
|
|
|
|
PageFrameNumber - Supplies the first physical page frame to insert.
|
|
|
|
EndingPageFrameNumberExclusive - Supplies the last physical page frame to
|
|
insert, not including the page itself.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
Environment:
|
|
|
|
Kernel mode.
|
|
|
|
--*/
|
|
{
|
|
MI_ASSERT_LOCK_MM();
|
|
|
|
while (PageFrameNumber < EndingPageFrameNumberExclusive) {
|
|
|
|
//
|
|
// MmAllocateContiguousMemorySpecifyCache searches for free pages from
|
|
// the highest page to the lowest page, so insert pages such that the
|
|
// lowest pages are at the front of the free lists. This avoids
|
|
// unnecessary page relocation in order to satisfy a contiguous memory
|
|
// allocation.
|
|
//
|
|
|
|
MiInsertPageInFreeList(PageFrameNumber, FALSE);
|
|
|
|
MmNumberOfPhysicalPages++;
|
|
|
|
PageFrameNumber++;
|
|
}
|
|
}
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiRemovePageFromFreeList(
|
|
IN PFN_NUMBER PageFrameNumber
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine removes the supplied physical page from the free page list
|
|
that it is currently attached to.
|
|
|
|
The returned page frame is not initialized.
|
|
|
|
Arguments:
|
|
|
|
PageFrameNumber - Supplies the physical page frame to remove.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
Environment:
|
|
|
|
Kernel mode, DISPATCH_LEVEL.
|
|
|
|
--*/
|
|
{
|
|
MMCOLOR Color;
|
|
USHORT PackedPfn;
|
|
PMMPFN PageFrame;
|
|
PMMPFNREGION PfnRegion;
|
|
PMMPFN LinkPageFrame;
|
|
|
|
MI_ASSERT_LOCK_MM();
|
|
|
|
ASSERT(PageFrameNumber <= MM_HIGHEST_PHYSICAL_PAGE);
|
|
|
|
Color = MiGetPfnColor(PageFrameNumber);
|
|
PackedPfn = MiPackFreePfn(PageFrameNumber);
|
|
|
|
PageFrame = MI_PFN_ELEMENT(PageFrameNumber);
|
|
PfnRegion = MI_PFN_REGION(PageFrameNumber);
|
|
|
|
ASSERT(PageFrame->Pte.Hard.Valid == 0);
|
|
ASSERT(PageFrame->Busy.Busy == 0);
|
|
|
|
//
|
|
// Detach the page from its backward link.
|
|
//
|
|
|
|
if (PageFrame->Free.PackedPfnBlink != MM_PACKED_PFN_NULL) {
|
|
|
|
LinkPageFrame = MiUnpackFreePfnElement(PageFrame->Free.PackedPfnBlink,
|
|
Color);
|
|
|
|
ASSERT(LinkPageFrame->Pte.Hard.Valid == 0);
|
|
ASSERT(LinkPageFrame->Busy.Busy == 0);
|
|
ASSERT(LinkPageFrame->Free.PackedPfnFlink == PackedPfn);
|
|
|
|
LinkPageFrame->Free.PackedPfnFlink = PageFrame->Free.PackedPfnFlink;
|
|
|
|
ASSERT(LinkPageFrame->Busy.Busy == 0);
|
|
|
|
} else {
|
|
|
|
//
|
|
// The page is the head of the list.
|
|
//
|
|
|
|
ASSERT(PfnRegion->FreePagesByColor[Color].PackedPfnFlink == PackedPfn);
|
|
PfnRegion->FreePagesByColor[Color].PackedPfnFlink =
|
|
PageFrame->Free.PackedPfnFlink;
|
|
}
|
|
|
|
//
|
|
// Detach the page from its forward link.
|
|
//
|
|
|
|
if (PageFrame->Free.PackedPfnFlink != MM_PACKED_PFN_NULL) {
|
|
|
|
LinkPageFrame = MiUnpackFreePfnElement(PageFrame->Free.PackedPfnFlink,
|
|
Color);
|
|
|
|
ASSERT(LinkPageFrame->Pte.Hard.Valid == 0);
|
|
ASSERT(LinkPageFrame->Busy.Busy == 0);
|
|
ASSERT(LinkPageFrame->Free.PackedPfnBlink == PackedPfn);
|
|
|
|
LinkPageFrame->Free.PackedPfnBlink = PageFrame->Free.PackedPfnBlink;
|
|
|
|
ASSERT(LinkPageFrame->Busy.Busy == 0);
|
|
|
|
} else {
|
|
|
|
//
|
|
// The page is the tail of the list.
|
|
//
|
|
|
|
ASSERT(PfnRegion->FreePagesByColor[Color].PackedPfnBlink == PackedPfn);
|
|
PfnRegion->FreePagesByColor[Color].PackedPfnBlink =
|
|
PageFrame->Free.PackedPfnBlink;
|
|
}
|
|
|
|
//
|
|
// Update the number of available pages in the region.
|
|
//
|
|
|
|
PfnRegion->AvailablePages--;
|
|
|
|
#ifdef CONSOLE_DEVKIT
|
|
//
|
|
// Update the number of available pages in the system if the page is from
|
|
// the retail PFN region or if the extra memory on a development kit is
|
|
// available for general use.
|
|
//
|
|
|
|
if (PfnRegion == &MmRetailPfnRegion || MmDeveloperKitMemoryReleased) {
|
|
MmAvailablePages--;
|
|
}
|
|
#else
|
|
//
|
|
// Update the number of available pages in the system.
|
|
//
|
|
|
|
MmAvailablePages--;
|
|
#endif
|
|
}
|
|
|
|
PFN_NUMBER
|
|
FASTCALL
|
|
MiRemoveAnyPageFromFreeList(
|
|
VOID
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine removes any physical page from the free page lists.
|
|
|
|
The returned page frame is not initialized.
|
|
|
|
Arguments:
|
|
|
|
None.
|
|
|
|
Return Value:
|
|
|
|
The allocated physical page frame number.
|
|
|
|
Environment:
|
|
|
|
Kernel mode, DISPATCH_LEVEL.
|
|
|
|
--*/
|
|
{
|
|
PMMPFNREGION PfnRegion;
|
|
MMCOLOR Color;
|
|
USHORT PackedPfn;
|
|
PFN_NUMBER PageFrameNumber;
|
|
PMMPFN PageFrame;
|
|
PMMPFN NewLinkPageFrame;
|
|
|
|
MI_ASSERT_LOCK_MM();
|
|
|
|
//
|
|
// Verify that there are pages available. The caller is responsible for
|
|
// first checking that a page can be allocated.
|
|
//
|
|
|
|
ASSERT(MmAvailablePages > 0);
|
|
|
|
//
|
|
// Look for the first region with a free page.
|
|
//
|
|
|
|
PfnRegion = &MmRetailPfnRegion;
|
|
|
|
#ifdef CONSOLE_DEVKIT
|
|
if (PfnRegion->AvailablePages == 0) {
|
|
PfnRegion = &MmDeveloperKitPfnRegion;
|
|
ASSERT(MmDeveloperKitMemoryReleased);
|
|
}
|
|
#endif
|
|
|
|
ASSERT(PfnRegion->AvailablePages > 0);
|
|
|
|
//
|
|
// Look for the first colored free list with a free page.
|
|
//
|
|
|
|
ASSERT(MmNextDefaultPageColor < MM_NUMBER_OF_COLORS);
|
|
|
|
for (Color = MmNextDefaultPageColor; NOTHING;
|
|
Color = ((Color + 1) & MM_NUMBER_OF_COLORS_MASK)) {
|
|
|
|
if (PfnRegion->FreePagesByColor[Color].PackedPfnFlink !=
|
|
MM_PACKED_PFN_NULL) {
|
|
|
|
PackedPfn = PfnRegion->FreePagesByColor[Color].PackedPfnFlink;
|
|
PageFrameNumber = MiUnpackFreePfn(PackedPfn, Color);
|
|
|
|
ASSERT(PageFrameNumber <= MM_HIGHEST_PHYSICAL_PAGE);
|
|
ASSERT(MI_PFN_REGION(PageFrameNumber) == PfnRegion);
|
|
|
|
PageFrame = MI_PFN_ELEMENT(PageFrameNumber);
|
|
|
|
ASSERT(PageFrame->Pte.Hard.Valid == 0);
|
|
ASSERT(PageFrame->Busy.Busy == 0);
|
|
|
|
//
|
|
// Unlink the page from the free list.
|
|
//
|
|
|
|
if (PageFrame->Free.PackedPfnFlink != MM_PACKED_PFN_NULL) {
|
|
|
|
NewLinkPageFrame = MiUnpackFreePfnElement(PageFrame->Free.PackedPfnFlink,
|
|
Color);
|
|
ASSERT(NewLinkPageFrame->Busy.Busy == 0);
|
|
|
|
ASSERT(NewLinkPageFrame->Free.PackedPfnBlink ==
|
|
PfnRegion->FreePagesByColor[Color].PackedPfnFlink);
|
|
NewLinkPageFrame->Free.PackedPfnBlink = MM_PACKED_PFN_NULL;
|
|
|
|
} else {
|
|
|
|
//
|
|
// The page is the tail of the list.
|
|
//
|
|
|
|
ASSERT(PfnRegion->FreePagesByColor[Color].PackedPfnBlink == PackedPfn);
|
|
PfnRegion->FreePagesByColor[Color].PackedPfnBlink = MM_PACKED_PFN_NULL;
|
|
}
|
|
|
|
ASSERT(PageFrame->Free.PackedPfnBlink == MM_PACKED_PFN_NULL);
|
|
PfnRegion->FreePagesByColor[Color].PackedPfnFlink =
|
|
PageFrame->Free.PackedPfnFlink;
|
|
|
|
break;
|
|
}
|
|
}
|
|
|
|
MmNextDefaultPageColor = ((Color + 1) & MM_NUMBER_OF_COLORS_MASK);
|
|
|
|
//
|
|
// Update the number of available pages in the region.
|
|
//
|
|
|
|
PfnRegion->AvailablePages--;
|
|
|
|
#ifdef CONSOLE_DEVKIT
|
|
//
|
|
// Update the number of available pages in the system if the page is from
|
|
// the retail PFN region or if the extra memory on a development kit is
|
|
// available for general use.
|
|
//
|
|
|
|
if (PfnRegion == &MmRetailPfnRegion || MmDeveloperKitMemoryReleased) {
|
|
MmAvailablePages--;
|
|
}
|
|
#else
|
|
//
|
|
// Update the number of available pages in the system.
|
|
//
|
|
|
|
MmAvailablePages--;
|
|
#endif
|
|
|
|
return PageFrameNumber;
|
|
}
|
|
|
|
PFN_NUMBER
|
|
FASTCALL
|
|
MiRemoveAnyPage(
|
|
IN MMPFN_BUSY_TYPE BusyType,
|
|
IN PMMPTE TargetPte
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine removes one page from the free list and initializes it as a
|
|
busy page with the supplied busy type.
|
|
|
|
The contents of the page are uninitialized.
|
|
|
|
Arguments:
|
|
|
|
BusyType - Usage code to place in allocated page frame.
|
|
|
|
TargetPte - Supplies the address of the PDE/PTE that will receive the page
|
|
frame.
|
|
|
|
Return Value:
|
|
|
|
The allocated physical page frame number.
|
|
|
|
Environment:
|
|
|
|
Kernel mode, DISPATCH_LEVEL.
|
|
|
|
--*/
|
|
{
|
|
PFN_NUMBER PageFrameNumber;
|
|
PMMPFN PageFrame;
|
|
|
|
ASSERT(BusyType < MmMaximumUsage);
|
|
|
|
MI_ASSERT_LOCK_MM();
|
|
|
|
//
|
|
// Allocate any page from a free list.
|
|
//
|
|
|
|
PageFrameNumber = MiRemoveAnyPageFromFreeList();
|
|
PageFrame = MI_PFN_ELEMENT(PageFrameNumber);
|
|
|
|
//
|
|
// Initialize the page frame as busy. Page tables created for the lower
|
|
// half of the address space (user space) are always allocated via
|
|
// MiRemoveZeroPage, so we don't need to handle that special case here.
|
|
//
|
|
|
|
ASSERT(BusyType != MmVirtualPageTableUsage);
|
|
|
|
PageFrame->Long = 0; // Also zeroes LockCount
|
|
PageFrame->Busy.Busy = 1;
|
|
PageFrame->Busy.BusyType = BusyType;
|
|
|
|
PageFrame->Busy.PteIndex = MiGetPteOffset(MiGetVirtualAddressMappedByPte(TargetPte));
|
|
|
|
//
|
|
// Increment the number of pages of this type.
|
|
//
|
|
|
|
MmAllocatedPagesByUsage[BusyType]++;
|
|
|
|
return PageFrameNumber;
|
|
}
|
|
|
|
PFN_NUMBER
|
|
FASTCALL
|
|
MiRemoveZeroPage(
|
|
IN MMPFN_BUSY_TYPE BusyType,
|
|
IN PMMPTE TargetPte
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine removes one page from the free list and initializes it as a
|
|
busy page with the supplied busy type.
|
|
|
|
The contents of the page are zeroed.
|
|
|
|
Arguments:
|
|
|
|
BusyType - Usage code to place in allocated page frame.
|
|
|
|
TargetPte - Supplies the address of the PDE/PTE that will receive the page
|
|
frame.
|
|
|
|
Return Value:
|
|
|
|
The allocated physical page frame number.
|
|
|
|
Environment:
|
|
|
|
Kernel mode, DISPATCH_LEVEL.
|
|
|
|
--*/
|
|
{
|
|
PFN_NUMBER PageFrameNumber;
|
|
PMMPFN PageFrame;
|
|
MMPTE TempPte;
|
|
|
|
ASSERT(BusyType < MmMaximumUsage);
|
|
|
|
MI_ASSERT_LOCK_MM();
|
|
|
|
//
|
|
// Allocate any page from a free list.
|
|
//
|
|
|
|
PageFrameNumber = MiRemoveAnyPageFromFreeList();
|
|
PageFrame = MI_PFN_ELEMENT(PageFrameNumber);
|
|
|
|
//
|
|
// Zero the page's contents by temporarily exposing it through the system
|
|
// RAM physical mapping.
|
|
//
|
|
// The below "nop" serves as a fake write barrier to ensure that
|
|
// MI_WRITE_PTE is not optimized away because we end up changing PageFrame
|
|
// below.
|
|
//
|
|
|
|
TempPte.Long = MiGetValidKernelPteBits();
|
|
TempPte.Hard.PageFrameNumber = PageFrameNumber;
|
|
|
|
MI_WRITE_PTE(&((volatile MMPFN*)PageFrame)->Pte, TempPte);
|
|
|
|
__asm nop;
|
|
|
|
RtlZeroMemory(MI_CONVERT_PFN_TO_PHYSICAL(PageFrameNumber), PAGE_SIZE);
|
|
|
|
//
|
|
// Initialize the page frame as busy.
|
|
//
|
|
|
|
PageFrame->Long = 0; // Also zeroes LockCount
|
|
PageFrame->Busy.Busy = 1;
|
|
PageFrame->Busy.BusyType = BusyType;
|
|
|
|
if (BusyType != MmVirtualPageTableUsage) {
|
|
PageFrame->Busy.PteIndex =
|
|
MiGetPteOffset(MiGetVirtualAddressMappedByPte(TargetPte));
|
|
}
|
|
|
|
//
|
|
// Increment the number of pages of this type.
|
|
//
|
|
|
|
MmAllocatedPagesByUsage[BusyType]++;
|
|
|
|
//
|
|
// Flush the system RAM physical mapping created above.
|
|
//
|
|
|
|
MI_FLUSH_VA(MI_CONVERT_PFN_TO_PHYSICAL(PageFrameNumber));
|
|
|
|
return PageFrameNumber;
|
|
}
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiRelocateBusyPage(
|
|
IN PFN_NUMBER PageFrameNumber
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine relocates the contents of the supplied physical page to another
|
|
physical page.
|
|
|
|
The original page frame is left in an uninitialized state.
|
|
|
|
Arguments:
|
|
|
|
PageFrameNumber - Supplies the physical page frame to relocate.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
Environment:
|
|
|
|
Kernel mode, DISPATCH_LEVEL.
|
|
|
|
--*/
|
|
{
|
|
PMMPFN PageFrame;
|
|
BOOLEAN FullTlbFlush;
|
|
PMMPTE PointerPte;
|
|
PMMPTE PointerPde;
|
|
PVOID CacheBuffer;
|
|
PFN_NUMBER NewPageFrameNumber;
|
|
PMMPFN NewPageFrame;
|
|
MMPTE TempPte;
|
|
|
|
MI_ASSERT_LOCK_MM();
|
|
|
|
PageFrame = MI_PFN_ELEMENT(PageFrameNumber);
|
|
|
|
ASSERT(PageFrame->Pte.Hard.Valid == 0);
|
|
ASSERT(PageFrame->Busy.Busy != 0);
|
|
ASSERT(PageFrame->Busy.BusyType != MmUnknownUsage);
|
|
ASSERT(PageFrame->Busy.LockCount == 0);
|
|
|
|
//
|
|
// Locate the pointer to the PTE for the page frame.
|
|
//
|
|
|
|
FullTlbFlush = FALSE;
|
|
|
|
switch (PageFrame->Busy.BusyType) {
|
|
|
|
case MmVirtualPageTableUsage:
|
|
//
|
|
// Busy.PteIndex isn't valid for a MmVirtualPageTableUsage page.
|
|
// Instead, scan through the PDE for the page table.
|
|
//
|
|
|
|
PointerPde = MiGetPdeAddress(MM_LOWEST_USER_ADDRESS);
|
|
|
|
for (;;) {
|
|
|
|
if ((PointerPde->Hard.Valid != 0) &&
|
|
(PointerPde->Hard.PageFrameNumber == PageFrameNumber)) {
|
|
PointerPte = PointerPde;
|
|
break;
|
|
}
|
|
|
|
PointerPde++;
|
|
|
|
//
|
|
// Verify that we haven't run off the end of the page directory.
|
|
//
|
|
|
|
ASSERT(PointerPde <= MiGetPdeAddress(MM_HIGHEST_USER_ADDRESS));
|
|
}
|
|
|
|
FullTlbFlush = TRUE;
|
|
|
|
break;
|
|
|
|
case MmSystemPageTableUsage:
|
|
//
|
|
// Busy.PteIndex is valid for a MmSystemPageTableUsage page, but
|
|
// it's relative to the page directory, not a page table.
|
|
//
|
|
|
|
PointerPte = (PMMPTE)PDE_BASE + PageFrame->Busy.PteIndex;
|
|
|
|
ASSERT(PointerPte->Hard.Valid != 0);
|
|
ASSERT(PointerPte->Hard.PageFrameNumber == PageFrameNumber);
|
|
|
|
FullTlbFlush = TRUE;
|
|
|
|
break;
|
|
|
|
case MmFsCacheUsage:
|
|
//
|
|
// FsCache.ElementIndex is the index into the FscElementArray which
|
|
// can be used to find the virtual address of the cache page.
|
|
//
|
|
|
|
ASSERT(PageFrame->FsCache.ElementIndex < FscNumberOfCachePages);
|
|
CacheBuffer = PAGE_ALIGN(FscElementArray[PageFrame->FsCache.ElementIndex].CacheBuffer);
|
|
PointerPte = MiGetPteAddress(CacheBuffer);
|
|
break;
|
|
|
|
case MmVirtualMemoryUsage:
|
|
case MmImageUsage:
|
|
//
|
|
// Busy.PteIndex is valid for a MmVirtualMemoryUsage page. Scan
|
|
// through each page table in the application virtual memory space.
|
|
//
|
|
|
|
PointerPde = MiGetPdeAddress(MM_LOWEST_USER_ADDRESS);
|
|
|
|
for (;;) {
|
|
|
|
if (PointerPde->Hard.Valid != 0) {
|
|
|
|
ASSERT(PointerPde->Hard.LargePage == 0);
|
|
|
|
PointerPte = (PMMPTE)MiGetVirtualAddressMappedByPte(PointerPde) +
|
|
PageFrame->Busy.PteIndex;
|
|
|
|
//
|
|
// The page may be protected as PAGE_NOACCESS or PAGE_GUARD,
|
|
// so we can't just check the Valid bit here.
|
|
//
|
|
|
|
if ((PointerPte->Long != 0) &&
|
|
(PointerPte->Hard.PageFrameNumber == PageFrameNumber)) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
PointerPde++;
|
|
|
|
//
|
|
// Verify that we haven't run off the end of the page directory.
|
|
// If we hit this, then that means that a page either has an
|
|
// invalid PTE index.
|
|
//
|
|
|
|
ASSERT(PointerPde <= MiGetPdeAddress(MM_HIGHEST_USER_ADDRESS));
|
|
}
|
|
break;
|
|
|
|
#ifdef DEVKIT
|
|
case MmDebuggerUsage:
|
|
#ifdef CONSOLE_DEVKIT
|
|
//
|
|
// Relocatable pages only exist in the first 64 megabytes of memory
|
|
// and debugger pages can only be allocated in this memory if the
|
|
// system only has 64 megabytes of memory.
|
|
//
|
|
|
|
ASSERT(MM_HIGHEST_PHYSICAL_PAGE < MM_64M_PHYSICAL_PAGE);
|
|
#endif
|
|
|
|
//
|
|
// Busy.PteIndex is valid for a MmDebuggerUsage page. Scan through
|
|
// each page table in the development kit PTE space.
|
|
//
|
|
|
|
PointerPde = MiGetPdeAddress(MM_DEVKIT_PTE_BASE);
|
|
|
|
for (;;) {
|
|
|
|
if (PointerPde->Hard.Valid != 0) {
|
|
|
|
ASSERT(PointerPde->Hard.LargePage == 0);
|
|
|
|
PointerPte = (PMMPTE)MiGetVirtualAddressMappedByPte(PointerPde) +
|
|
PageFrame->Busy.PteIndex;
|
|
|
|
if ((PointerPte->Hard.Valid != 0) &&
|
|
(PointerPte->Hard.PageFrameNumber == PageFrameNumber)) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
PointerPde++;
|
|
|
|
//
|
|
// Verify that we haven't run off the end of the page directory.
|
|
// If we hit this, then that means that a page either has an
|
|
// invalid PTE index.
|
|
//
|
|
|
|
ASSERT(PointerPde <= MiGetPdeAddress(MM_DEVKIT_PTE_END));
|
|
}
|
|
break;
|
|
#endif
|
|
|
|
default:
|
|
#ifdef CONSOLE_DEVKIT
|
|
//
|
|
// For non-DEVKIT builds or builds with more than 64 megabytes of
|
|
// memory, make sure that we don't hit a debugger page.
|
|
//
|
|
|
|
ASSERT(PageFrame->Busy.BusyType != MmDebuggerUsage);
|
|
#endif
|
|
|
|
//
|
|
// Busy.PteIndex is valid for all other types. Scan through each
|
|
// page table in the system PTE space.
|
|
//
|
|
|
|
PointerPde = MiGetPdeAddress(MM_SYSTEM_PTE_BASE);
|
|
|
|
for (;;) {
|
|
|
|
if (PointerPde->Hard.Valid != 0) {
|
|
|
|
ASSERT(PointerPde->Hard.LargePage == 0);
|
|
|
|
PointerPte = (PMMPTE)MiGetVirtualAddressMappedByPte(PointerPde) +
|
|
PageFrame->Busy.PteIndex;
|
|
|
|
if ((PointerPte->Hard.Valid != 0) &&
|
|
(PointerPte->Hard.PageFrameNumber == PageFrameNumber)) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
PointerPde++;
|
|
|
|
//
|
|
// Verify that we haven't run off the end of the page directory.
|
|
// If we hit this, then that means that a page either has an
|
|
// invalid PTE index.
|
|
//
|
|
|
|
ASSERT(PointerPde <= MiGetPdeAddress(MM_SYSTEM_PTE_END));
|
|
}
|
|
break;
|
|
}
|
|
|
|
//
|
|
// Allocate another page to hold the contents of the page. Attempt to
|
|
// obtain the same color page so that virtual buffers are still spaced
|
|
// nicely in the cache.
|
|
//
|
|
|
|
MmNextDefaultPageColor = MiGetPfnColor(PageFrameNumber);
|
|
NewPageFrameNumber = MiRemoveAnyPageFromFreeList();
|
|
NewPageFrame = MI_PFN_ELEMENT(NewPageFrameNumber);
|
|
|
|
//
|
|
// Temporarily expose the new page through the system RAM physical mapping
|
|
// so that we can copy the page from the old address to the new address.
|
|
//
|
|
|
|
TempPte.Long = MiGetValidKernelPteBits();
|
|
TempPte.Hard.PageFrameNumber = NewPageFrameNumber;
|
|
|
|
MI_WRITE_PTE(&((volatile MMPFN*)NewPageFrame)->Pte, TempPte);
|
|
|
|
//
|
|
// Build the PTE contents for the virtual address to point at the new page.
|
|
//
|
|
|
|
TempPte = *PointerPte;
|
|
ASSERT(PointerPte->Hard.PageFrameNumber == PageFrameNumber);
|
|
TempPte.Hard.PageFrameNumber = NewPageFrameNumber;
|
|
|
|
//
|
|
// Invoke a helper to copy the contents of the page from the old page to the
|
|
// new page, to update the page tables, and to flush the appropriate TLB
|
|
// entries. This is done in an assembly routine in order to ensure the
|
|
// stack is updated properly if we're relocating the current stack's pages.
|
|
//
|
|
|
|
MiRelocateBusyPageHelper(MiGetVirtualAddressMappedByPte(PointerPte),
|
|
MI_CONVERT_PFN_TO_PHYSICAL(NewPageFrameNumber), PointerPte, TempPte,
|
|
FullTlbFlush);
|
|
|
|
//
|
|
// Copy the page frame attributes from the old page frame to the new.
|
|
//
|
|
|
|
NewPageFrame->Long = PageFrame->Long;
|
|
|
|
//
|
|
// Flush the system RAM physical mapping created above.
|
|
//
|
|
|
|
MI_FLUSH_VA(MI_CONVERT_PFN_TO_PHYSICAL(NewPageFrameNumber));
|
|
|
|
//
|
|
// If caching is disabled or is write combined for the page's old virtual
|
|
// address, then force a flush of the TLB and cache now to ensure coherency.
|
|
//
|
|
|
|
if (TempPte.Hard.WriteThrough) {
|
|
KeFlushCurrentTbAndInvalidateAllCaches();
|
|
}
|
|
}
|
|
|
|
VOID
|
|
FASTCALL
|
|
MiReleasePageOwnership(
|
|
IN PFN_NUMBER PageFrameNumber
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine is called when the original use of an allocated page is
|
|
complete.
|
|
|
|
Arguments:
|
|
|
|
PageFrameNumber - Supplies the physical page frame to release.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
Environment:
|
|
|
|
Kernel mode, DISPATCH_LEVEL.
|
|
|
|
--*/
|
|
{
|
|
PMMPFN PageFrame;
|
|
|
|
MI_ASSERT_LOCK_MM();
|
|
|
|
ASSERT(PageFrameNumber <= MM_HIGHEST_PHYSICAL_PAGE);
|
|
|
|
PageFrame = MI_PFN_ELEMENT(PageFrameNumber);
|
|
|
|
ASSERT(PageFrame->Pte.Hard.Valid == 0);
|
|
ASSERT(PageFrame->Busy.Busy != 0);
|
|
ASSERT(PageFrame->Busy.BusyType != MmUnknownUsage);
|
|
ASSERT(PageFrame->Busy.BusyType < MmMaximumUsage);
|
|
|
|
//
|
|
// If the page is still locked for I/O, then this is a bug in the client
|
|
// code. Pages should never be released with pending I/O.
|
|
//
|
|
|
|
ASSERT(PageFrame->Busy.LockCount == 0);
|
|
|
|
//
|
|
// Decrement the number of pages of this type.
|
|
//
|
|
|
|
MmAllocatedPagesByUsage[PageFrame->Busy.BusyType]--;
|
|
|
|
//
|
|
// Insert the page at the head of the free lists so that this page is
|
|
// more likely to be allocated before a previously pinned page.
|
|
//
|
|
|
|
MiInsertPageInFreeList(PageFrameNumber, TRUE);
|
|
}
|
|
|
|
#ifdef CONSOLE_DEVKIT
|
|
|
|
VOID
|
|
MmReleaseDeveloperKitMemory(
|
|
VOID
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine places all of the available pages in the extra memory PFN
|
|
region back in the general available pages pool. All of the other virtual
|
|
memory manager services can start using these pages.
|
|
|
|
Arguments:
|
|
|
|
None.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
Environment:
|
|
|
|
Kernel mode.
|
|
|
|
--*/
|
|
{
|
|
KIRQL OldIrql;
|
|
|
|
MI_LOCK_MM(&OldIrql);
|
|
|
|
MmAvailablePages += MmDeveloperKitPfnRegion.AvailablePages;
|
|
|
|
MmDeveloperKitMemoryReleased = TRUE;
|
|
|
|
MI_UNLOCK_MM(OldIrql);
|
|
}
|
|
|
|
PFN_NUMBER
|
|
FASTCALL
|
|
MiRemoveDebuggerPage(
|
|
IN MMPFN_BUSY_TYPE BusyType,
|
|
IN PMMPTE TargetPte
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine removes one page from the free list and initializes it as a
|
|
busy page with the supplied busy type.
|
|
|
|
The contents of the page are zeroed.
|
|
|
|
Arguments:
|
|
|
|
BusyType - Ignored. MmDebuggerUsage is used for all pages allocated by
|
|
this routine.
|
|
|
|
TargetPte - Supplies the address of the PDE/PTE that will receive the page
|
|
frame.
|
|
|
|
Return Value:
|
|
|
|
The allocated physical page frame number.
|
|
|
|
Environment:
|
|
|
|
Kernel mode, DISPATCH_LEVEL.
|
|
|
|
--*/
|
|
{
|
|
MMCOLOR Color;
|
|
USHORT PackedPfn;
|
|
PFN_NUMBER PageFrameNumber;
|
|
PMMPFN PageFrame;
|
|
MMPTE TempPte;
|
|
|
|
MI_ASSERT_LOCK_MM();
|
|
|
|
//
|
|
// Verify that there are pages available. The caller is responsible for
|
|
// first checking that a page can be allocated.
|
|
//
|
|
|
|
ASSERT(MmDeveloperKitPfnRegion.AvailablePages > 0);
|
|
|
|
//
|
|
// Look for the first colored free list with a free page.
|
|
//
|
|
|
|
ASSERT(MmNextDefaultPageColor < MM_NUMBER_OF_COLORS);
|
|
|
|
for (Color = MmNextDefaultPageColor; NOTHING;
|
|
Color = ((Color + 1) & MM_NUMBER_OF_COLORS_MASK)) {
|
|
|
|
if (MmDeveloperKitPfnRegion.FreePagesByColor[Color].PackedPfnFlink !=
|
|
MM_PACKED_PFN_NULL) {
|
|
|
|
PackedPfn = MmDeveloperKitPfnRegion.FreePagesByColor[Color].PackedPfnFlink;
|
|
PageFrameNumber = MiUnpackFreePfn(PackedPfn, Color);
|
|
|
|
//
|
|
// MiRemoveAnyPage is optimized for speed and detaches the page from
|
|
// the free list itself knowing that the page is at the head of the
|
|
// list. Here, we optimize for size and delegate the list
|
|
// detachment to a common routine.
|
|
//
|
|
|
|
MiRemovePageFromFreeList(PageFrameNumber);
|
|
|
|
break;
|
|
}
|
|
}
|
|
|
|
MmNextDefaultPageColor = ((Color + 1) & MM_NUMBER_OF_COLORS_MASK);
|
|
|
|
PageFrame = MI_PFN_ELEMENT(PageFrameNumber);
|
|
|
|
//
|
|
// Zero the page's contents by temporarily exposing it through the system
|
|
// RAM physical mapping.
|
|
//
|
|
// The below "nop" serves as a fake write barrier to ensure that
|
|
// MI_WRITE_PTE is not optimized away because we end up changing PageFrame
|
|
// below.
|
|
//
|
|
|
|
TempPte.Long = MiGetValidKernelPteBits();
|
|
TempPte.Hard.PageFrameNumber = PageFrameNumber;
|
|
|
|
MI_WRITE_PTE(&((volatile MMPFN*)PageFrame)->Pte, TempPte);
|
|
|
|
__asm nop;
|
|
|
|
RtlZeroMemory(MI_CONVERT_PFN_TO_PHYSICAL(PageFrameNumber), PAGE_SIZE);
|
|
|
|
//
|
|
// Initialize the page frame as busy.
|
|
//
|
|
|
|
PageFrame->Long = 0; // Also zeroes LockCount
|
|
PageFrame->Busy.Busy = 1;
|
|
PageFrame->Busy.BusyType = MmDebuggerUsage;
|
|
PageFrame->Busy.PteIndex =
|
|
MiGetPteOffset(MiGetVirtualAddressMappedByPte(TargetPte));
|
|
|
|
//
|
|
// Increment the number of pages of this type.
|
|
//
|
|
|
|
MmAllocatedPagesByUsage[MmDebuggerUsage]++;
|
|
|
|
//
|
|
// Flush the system RAM physical mapping created above.
|
|
//
|
|
|
|
MI_FLUSH_VA(MI_CONVERT_PFN_TO_PHYSICAL(PageFrameNumber));
|
|
|
|
return PageFrameNumber;
|
|
}
|
|
|
|
#endif
|
|
|
|
#if DBG
|
|
|
|
VOID
|
|
MiAssertPfnRegionFreeListsValid(
|
|
IN PMMPFNREGION PfnRegion
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine analyzes the free lists of a PFN region to verify that they are
|
|
structurally correct.
|
|
|
|
Arguments:
|
|
|
|
PfnRegion - Supplies the PFN region to verify.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
Environment:
|
|
|
|
Kernel mode, DISPATCH_LEVEL.
|
|
|
|
--*/
|
|
{
|
|
PFN_COUNT FreePagesFound;
|
|
MMCOLOR Color;
|
|
USHORT PackedPfnBlink;
|
|
USHORT PackedPfn;
|
|
PFN_NUMBER PageFrameNumber;
|
|
PMMPFN PageFrame;
|
|
|
|
MI_ASSERT_LOCK_MM();
|
|
|
|
FreePagesFound = 0;
|
|
|
|
for (Color = 0; Color < MM_NUMBER_OF_COLORS; Color++) {
|
|
|
|
PackedPfnBlink = MM_PACKED_PFN_NULL;
|
|
PackedPfn = PfnRegion->FreePagesByColor[Color].PackedPfnFlink;
|
|
|
|
while (PackedPfn != MM_PACKED_PFN_NULL) {
|
|
|
|
ASSERT((PackedPfn & 1) == 0);
|
|
|
|
PageFrameNumber = MiUnpackFreePfn(PackedPfn, Color);
|
|
|
|
ASSERT(PageFrameNumber <= MM_HIGHEST_PHYSICAL_PAGE);
|
|
ASSERT(MI_PFN_REGION(PageFrameNumber) == PfnRegion);
|
|
|
|
PageFrame = MI_PFN_ELEMENT(PageFrameNumber);
|
|
|
|
ASSERT(PageFrame->Pte.Hard.Valid == 0);
|
|
ASSERT(PageFrame->Busy.Busy == 0);
|
|
ASSERT(PageFrame->Free.PackedPfnBlink == PackedPfnBlink);
|
|
|
|
FreePagesFound++;
|
|
|
|
PackedPfnBlink = PackedPfn;
|
|
PackedPfn = PageFrame->Free.PackedPfnFlink;
|
|
|
|
ASSERT(FreePagesFound <= MmAvailablePages);
|
|
}
|
|
|
|
ASSERT(PfnRegion->FreePagesByColor[Color].PackedPfnBlink == PackedPfnBlink);
|
|
}
|
|
|
|
ASSERT(PfnRegion->AvailablePages == FreePagesFound);
|
|
}
|
|
|
|
VOID
|
|
MiAssertFreeListsValid(
|
|
VOID
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine analyzes the free lists to verify that they are structurally
|
|
correct.
|
|
|
|
Arguments:
|
|
|
|
None.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
Environment:
|
|
|
|
Kernel mode, DISPATCH_LEVEL.
|
|
|
|
--*/
|
|
{
|
|
MiAssertPfnRegionFreeListsValid(&MmRetailPfnRegion);
|
|
#ifdef CONSOLE_DEVKIT
|
|
MiAssertPfnRegionFreeListsValid(&MmDeveloperKitPfnRegion);
|
|
#endif
|
|
}
|
|
|
|
#endif // DBG
|