2437 lines
67 KiB
C
2437 lines
67 KiB
C
/*++
|
|
|
|
Copyright (c) 1989 Microsoft Corporation
|
|
|
|
Module Name:
|
|
|
|
forksup.c
|
|
|
|
Abstract:
|
|
|
|
This module contains the routines which support the POSIX fork operation.
|
|
|
|
Author:
|
|
|
|
Lou Perazzoli (loup) 22-Jul-1989
|
|
Landy Wang (landyw) 02-June-1997
|
|
|
|
Revision History:
|
|
|
|
--*/
|
|
|
|
#include "mi.h"
|
|
|
|
VOID
|
|
MiUpPfnReferenceCount (
|
|
IN PFN_NUMBER Page,
|
|
IN USHORT Count
|
|
);
|
|
|
|
VOID
|
|
MiDownPfnReferenceCount (
|
|
IN PFN_NUMBER Page,
|
|
IN USHORT Count
|
|
);
|
|
|
|
VOID
|
|
MiUpControlAreaRefs (
|
|
IN PCONTROL_AREA ControlArea
|
|
);
|
|
|
|
ULONG
|
|
MiDoneWithThisPageGetAnother (
|
|
IN PPFN_NUMBER PageFrameIndex,
|
|
IN PMMPTE PointerPde,
|
|
IN PEPROCESS CurrentProcess
|
|
);
|
|
|
|
VOID
|
|
MiUpForkPageShareCount(
|
|
IN PMMPFN PfnForkPtePage
|
|
);
|
|
|
|
VOID
|
|
MiUpCloneProcessRefCount (
|
|
IN PMMCLONE_DESCRIPTOR Clone
|
|
);
|
|
|
|
VOID
|
|
MiUpCloneProtoRefCount (
|
|
IN PMMCLONE_BLOCK CloneProto,
|
|
IN PEPROCESS CurrentProcess
|
|
);
|
|
|
|
ULONG
|
|
MiHandleForkTransitionPte (
|
|
IN PMMPTE PointerPte,
|
|
IN PMMPTE PointerNewPte,
|
|
IN PMMCLONE_BLOCK ForkProtoPte
|
|
);
|
|
|
|
VOID
|
|
MiDownShareCountFlushEntireTb (
|
|
IN PFN_NUMBER PageFrameIndex
|
|
);
|
|
|
|
VOID
|
|
MiBuildForkPageTable(
|
|
IN PFN_NUMBER PageFrameIndex,
|
|
IN PMMPTE PointerPde,
|
|
IN PMMPTE PointerNewPde,
|
|
IN PFN_NUMBER PdePhysicalPage,
|
|
IN PMMPFN PfnPdPage
|
|
);
|
|
|
|
VOID
|
|
MiRetrievePageDirectoryFrames(
|
|
IN PFN_NUMBER RootPhysicalPage,
|
|
OUT PPFN_NUMBER PageDirectoryFrames
|
|
);
|
|
|
|
#define MM_FORK_SUCCEEDED 0
|
|
#define MM_FORK_FAILED 1
|
|
|
|
#ifdef ALLOC_PRAGMA
|
|
#pragma alloc_text(PAGE,MiCloneProcessAddressSpace)
|
|
#endif
|
|
|
|
|
|
NTSTATUS
|
|
MiCloneProcessAddressSpace (
|
|
IN PEPROCESS ProcessToClone,
|
|
IN PEPROCESS ProcessToInitialize,
|
|
IN PFN_NUMBER RootPhysicalPage,
|
|
IN PFN_NUMBER HyperPhysicalPage
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine stands on its head to produce a copy of the specified
|
|
process's address space in the process to initialize. This
|
|
is done by examining each virtual address descriptor's inherit
|
|
attributes. If the pages described by the VAD should be inherited,
|
|
each PTE is examined and copied into the new address space.
|
|
|
|
For private pages, fork prototype PTEs are constructed and the pages
|
|
become shared, copy-on-write, between the two processes.
|
|
|
|
|
|
Arguments:
|
|
|
|
ProcessToClone - Supplies the process whose address space should be
|
|
cloned.
|
|
|
|
ProcessToInitialize - Supplies the process whose address space is to
|
|
be created.
|
|
|
|
RootPhysicalPage - Supplies the physical page number of the top level
|
|
page (parent on 64-bit systems) directory
|
|
of the process to initialize.
|
|
|
|
HyperPhysicalPage - Supplies the physical page number of the page table
|
|
page which maps hyperspace for the process to
|
|
initialize. This is only needed for 32-bit systems.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
Environment:
|
|
|
|
Kernel mode, APCs disabled.
|
|
|
|
--*/
|
|
|
|
{
|
|
PFN_NUMBER PpePhysicalPage;
|
|
PFN_NUMBER PdePhysicalPage;
|
|
PEPROCESS CurrentProcess;
|
|
PMMPTE PdeBase;
|
|
PMMCLONE_HEADER CloneHeader;
|
|
PMMCLONE_BLOCK CloneProtos;
|
|
PMMCLONE_DESCRIPTOR CloneDescriptor;
|
|
PMMVAD NewVad;
|
|
PMMVAD Vad;
|
|
PMMVAD NextVad;
|
|
PMMVAD *VadList;
|
|
PMMVAD FirstNewVad;
|
|
PMMCLONE_DESCRIPTOR *CloneList;
|
|
PMMCLONE_DESCRIPTOR FirstNewClone;
|
|
PMMCLONE_DESCRIPTOR Clone;
|
|
PMMCLONE_DESCRIPTOR NextClone;
|
|
PMMCLONE_DESCRIPTOR NewClone;
|
|
ULONG Attached;
|
|
ULONG CloneFailed;
|
|
ULONG VadInsertFailed;
|
|
WSLE_NUMBER WorkingSetIndex;
|
|
PVOID VirtualAddress;
|
|
NTSTATUS status;
|
|
PMMPFN Pfn2;
|
|
PMMPFN PfnPdPage;
|
|
MMPTE TempPte;
|
|
MMPTE PteContents;
|
|
#if defined (_X86PAE_)
|
|
PMDL MdlPageDirectory;
|
|
PPFN_NUMBER MdlPageFrames;
|
|
PFN_NUMBER PageDirectoryFrames[PD_PER_SYSTEM];
|
|
PFN_NUMBER MdlHackPageDirectory[(sizeof(MDL)/sizeof(PFN_NUMBER)) + PD_PER_SYSTEM];
|
|
#endif
|
|
PFN_NUMBER MdlPage;
|
|
PFN_NUMBER MdlDirPage;
|
|
PMMPTE PointerPte;
|
|
PMMPTE PointerPde;
|
|
PMMPTE PointerPpe;
|
|
PMMPTE LastPte;
|
|
PMMPTE PointerNewPte;
|
|
PMMPTE NewPteMappedAddress;
|
|
PMMPTE PointerNewPde;
|
|
PLIST_ENTRY NextEntry;
|
|
PMI_PHYSICAL_VIEW PhysicalView;
|
|
PFN_NUMBER PageFrameIndex;
|
|
PFN_NUMBER PageDirFrameIndex;
|
|
PMMCLONE_BLOCK ForkProtoPte;
|
|
PMMCLONE_BLOCK CloneProto;
|
|
PMMCLONE_BLOCK LockedForkPte;
|
|
PMMPTE ContainingPte;
|
|
ULONG NumberOfForkPtes;
|
|
PFN_NUMBER NumberOfPrivatePages;
|
|
PFN_NUMBER PageTablePage;
|
|
SIZE_T TotalPagedPoolCharge;
|
|
SIZE_T TotalNonPagedPoolCharge;
|
|
PMMPFN PfnForkPtePage;
|
|
PVOID UsedPageTableEntries;
|
|
ULONG ReleasedWorkingSetMutex;
|
|
ULONG FirstTime;
|
|
ULONG Waited;
|
|
ULONG i;
|
|
ULONG PpePdeOffset;
|
|
#if defined (_WIN64)
|
|
PVOID UsedPageDirectoryEntries;
|
|
PMMPTE PointerNewPpe;
|
|
PMMPTE PpeBase;
|
|
PMMPFN PfnPpPage;
|
|
#else
|
|
PMMWSL HyperBase;
|
|
#endif
|
|
|
|
PageTablePage = 2;
|
|
NumberOfForkPtes = 0;
|
|
Attached = FALSE;
|
|
PageFrameIndex = (PFN_NUMBER)-1;
|
|
PageDirFrameIndex = (PFN_NUMBER)-1;
|
|
|
|
#if DBG
|
|
if (MmDebug & MM_DBG_FORK) {
|
|
DbgPrint("beginning clone operation process to clone = %lx\n",
|
|
ProcessToClone);
|
|
}
|
|
#endif //DBG
|
|
|
|
PAGED_CODE();
|
|
|
|
if (ProcessToClone != PsGetCurrentProcess()) {
|
|
Attached = TRUE;
|
|
KeAttachProcess (&ProcessToClone->Pcb);
|
|
}
|
|
|
|
#if defined (_X86PAE_)
|
|
MiRetrievePageDirectoryFrames (RootPhysicalPage, PageDirectoryFrames);
|
|
#endif
|
|
|
|
CurrentProcess = ProcessToClone;
|
|
|
|
|
|
// Get the working set mutex and the address creation mutex
|
|
// of the process to clone. This prevents page faults while we
|
|
// are examining the address map and prevents virtual address space
|
|
// from being created or deleted.
|
|
|
|
|
|
LOCK_ADDRESS_SPACE (CurrentProcess);
|
|
|
|
|
|
// Write-watch VAD bitmaps are not currently duplicated
|
|
// so fork is not allowed.
|
|
|
|
|
|
if (CurrentProcess->Vm.u.Flags.WriteWatch == 1) {
|
|
status = STATUS_INVALID_PAGE_PROTECTION;
|
|
goto ErrorReturn1;
|
|
}
|
|
|
|
|
|
// Check for AWE regions as they are not duplicated so fork is not allowed.
|
|
// Note that since this is a readonly list walk, the address space mutex
|
|
// is sufficient to synchronize properly.
|
|
|
|
|
|
NextEntry = CurrentProcess->PhysicalVadList.Flink;
|
|
while (NextEntry != &CurrentProcess->PhysicalVadList) {
|
|
|
|
PhysicalView = CONTAINING_RECORD(NextEntry,
|
|
MI_PHYSICAL_VIEW,
|
|
ListEntry);
|
|
|
|
if (PhysicalView->Vad->u.VadFlags.UserPhysicalPages == 1) {
|
|
status = STATUS_INVALID_PAGE_PROTECTION;
|
|
goto ErrorReturn1;
|
|
}
|
|
|
|
NextEntry = NextEntry->Flink;
|
|
}
|
|
|
|
|
|
// Make sure the address space was not deleted, if so, return an error.
|
|
|
|
|
|
if (CurrentProcess->AddressSpaceDeleted != 0) {
|
|
status = STATUS_PROCESS_IS_TERMINATING;
|
|
goto ErrorReturn1;
|
|
}
|
|
|
|
// Attempt to acquire the needed pool before starting the
|
|
// clone operation, this allows an easier failure path in the case of insufficient system resources.
|
|
|
|
NumberOfPrivatePages = CurrentProcess->NumberOfPrivatePages;
|
|
CloneProtos = ExAllocatePoolWithTag (PagedPool, sizeof(MMCLONE_BLOCK) * NumberOfPrivatePages, 'lCmM');
|
|
if (CloneProtos == NULL) {
|
|
status = STATUS_INSUFFICIENT_RESOURCES;
|
|
goto ErrorReturn1;
|
|
}
|
|
|
|
CloneHeader = ExAllocatePoolWithTag (NonPagedPool, sizeof(MMCLONE_HEADER), ' mM');
|
|
if (CloneHeader == NULL) {
|
|
status = STATUS_INSUFFICIENT_RESOURCES;
|
|
goto ErrorReturn2;
|
|
}
|
|
|
|
CloneDescriptor = ExAllocatePoolWithTag (NonPagedPool, sizeof(MMCLONE_DESCRIPTOR), ' mM');
|
|
if (CloneDescriptor == NULL) {
|
|
status = STATUS_INSUFFICIENT_RESOURCES;
|
|
goto ErrorReturn3;
|
|
}
|
|
|
|
Vad = MiGetFirstVad (CurrentProcess);
|
|
VadList = &FirstNewVad;
|
|
|
|
while (Vad != (PMMVAD)NULL) {
|
|
// If the VAD does not go to the child, ignore it.
|
|
if ((Vad->u.VadFlags.UserPhysicalPages == 0) &&
|
|
((Vad->u.VadFlags.PrivateMemory == 1) || (Vad->u2.VadFlags2.Inherit == MM_VIEW_SHARE))) {
|
|
NewVad = ExAllocatePoolWithTag (NonPagedPool, sizeof(MMVAD), ' daV');
|
|
if (NewVad == NULL) {
|
|
// Unable to allocate pool for all the VADs. Deallocate all VADs and other pool obtained so far.
|
|
*VadList = (PMMVAD)NULL;
|
|
status = STATUS_INSUFFICIENT_RESOURCES;
|
|
goto ErrorReturn4;
|
|
}
|
|
*VadList = NewVad;
|
|
VadList = &NewVad->Parent;
|
|
}
|
|
Vad = MiGetNextVad (Vad);
|
|
}
|
|
|
|
|
|
// Terminate list of VADs for new process.
|
|
|
|
|
|
*VadList = (PMMVAD)NULL;
|
|
|
|
|
|
// Charge the current process the quota for the paged and nonpaged
|
|
// global structures. This consists of the array of clone blocks
|
|
// in paged pool and the clone header in non-paged pool.
|
|
|
|
|
|
try {
|
|
PsChargePoolQuota (CurrentProcess, PagedPool, sizeof(MMCLONE_BLOCK) *
|
|
NumberOfPrivatePages);
|
|
PageTablePage = 1;
|
|
PsChargePoolQuota (CurrentProcess, NonPagedPool, sizeof(MMCLONE_HEADER));
|
|
PageTablePage = 0;
|
|
|
|
} except (EXCEPTION_EXECUTE_HANDLER) {
|
|
|
|
|
|
// Unable to charge quota for the clone blocks.
|
|
|
|
|
|
status = GetExceptionCode();
|
|
goto ErrorReturn4;
|
|
}
|
|
|
|
LOCK_WS (CurrentProcess);
|
|
|
|
ASSERT (CurrentProcess->ForkInProgress == NULL);
|
|
|
|
|
|
// Indicate to the pager that the current process is being
|
|
// forked. This blocks other threads in that process from
|
|
// modifying clone blocks counts and contents.
|
|
|
|
|
|
CurrentProcess->ForkInProgress = PsGetCurrentThread();
|
|
|
|
#if defined (_WIN64)
|
|
|
|
|
|
// Increment the reference count for the pages which are being "locked"
|
|
// in MDLs. This prevents the page from being reused while it is
|
|
// being double mapped. Note we have a count of 3 below because in addition
|
|
// to the PPE, we also set up a initial dummy PDE and PTE.
|
|
|
|
|
|
MiUpPfnReferenceCount (RootPhysicalPage, 3);
|
|
|
|
|
|
// Map the page directory parent page into the system address
|
|
// space. This is accomplished by building an MDL to describe the
|
|
// Page directory parent page.
|
|
|
|
|
|
PpeBase = (PMMPTE)MiMapSinglePage (NULL,
|
|
RootPhysicalPage,
|
|
MmCached,
|
|
HighPagePriority);
|
|
|
|
if (PpeBase == NULL) {
|
|
MiDownPfnReferenceCount (RootPhysicalPage, 3);
|
|
CurrentProcess->ForkInProgress = NULL;
|
|
UNLOCK_WS (CurrentProcess);
|
|
status = STATUS_INSUFFICIENT_RESOURCES;
|
|
goto ErrorReturn4;
|
|
}
|
|
|
|
PfnPpPage = MI_PFN_ELEMENT (RootPhysicalPage);
|
|
|
|
#else
|
|
|
|
#if !defined (_X86PAE_)
|
|
MiUpPfnReferenceCount (RootPhysicalPage, 1);
|
|
#endif
|
|
|
|
#endif
|
|
|
|
|
|
// Initialize a page directory map so it can be
|
|
// unlocked in the loop and the end of the loop without
|
|
// any testing to see if has a valid value the first time through.
|
|
// Note this is a dummy map for 64-bit systems and a real one for 32-bit.
|
|
|
|
|
|
#if !defined (_X86PAE_)
|
|
|
|
MdlDirPage = RootPhysicalPage;
|
|
|
|
PdePhysicalPage = RootPhysicalPage;
|
|
|
|
PdeBase = (PMMPTE)MiMapSinglePage (NULL,
|
|
MdlDirPage,
|
|
MmCached,
|
|
HighPagePriority);
|
|
|
|
if (PdeBase == NULL) {
|
|
#if defined (_WIN64)
|
|
MiDownPfnReferenceCount (RootPhysicalPage, 3);
|
|
MiUnmapSinglePage (PpeBase);
|
|
#else
|
|
MiDownPfnReferenceCount (RootPhysicalPage, 1);
|
|
#endif
|
|
CurrentProcess->ForkInProgress = NULL;
|
|
UNLOCK_WS (CurrentProcess);
|
|
status = STATUS_INSUFFICIENT_RESOURCES;
|
|
goto ErrorReturn4;
|
|
}
|
|
|
|
#else
|
|
|
|
|
|
// All 4 page directory pages need to be mapped for PAE so the heavyweight
|
|
// mapping must be used.
|
|
|
|
|
|
MdlPageDirectory = (PMDL)&MdlHackPageDirectory[0];
|
|
|
|
MmInitializeMdl(MdlPageDirectory, (PVOID)PDE_BASE, PD_PER_SYSTEM * PAGE_SIZE);
|
|
MdlPageDirectory->MdlFlags |= MDL_PAGES_LOCKED;
|
|
|
|
MdlPageFrames = (PPFN_NUMBER)(MdlPageDirectory + 1);
|
|
|
|
for (i = 0; i < PD_PER_SYSTEM; i += 1) {
|
|
*(MdlPageFrames + i) = PageDirectoryFrames[i];
|
|
MiUpPfnReferenceCount (PageDirectoryFrames[i], 1);
|
|
}
|
|
|
|
PdePhysicalPage = RootPhysicalPage;
|
|
|
|
PdeBase = (PMMPTE)MmMapLockedPagesSpecifyCache (MdlPageDirectory,
|
|
KernelMode,
|
|
MmCached,
|
|
NULL,
|
|
FALSE,
|
|
HighPagePriority);
|
|
|
|
if (PdeBase == NULL) {
|
|
for (i = 0; i < PD_PER_SYSTEM; i += 1) {
|
|
MiDownPfnReferenceCount (PageDirectoryFrames[i], 1);
|
|
}
|
|
CurrentProcess->ForkInProgress = NULL;
|
|
UNLOCK_WS (CurrentProcess);
|
|
status = STATUS_INSUFFICIENT_RESOURCES;
|
|
goto ErrorReturn4;
|
|
}
|
|
|
|
#endif
|
|
|
|
PfnPdPage = MI_PFN_ELEMENT (RootPhysicalPage);
|
|
|
|
#if !defined (_WIN64)
|
|
|
|
|
|
// Map hyperspace so target UsedPageTable entries can be incremented.
|
|
|
|
|
|
MiUpPfnReferenceCount (HyperPhysicalPage, 2);
|
|
|
|
HyperBase = (PMMWSL)MiMapSinglePage (NULL,
|
|
HyperPhysicalPage,
|
|
MmCached,
|
|
HighPagePriority);
|
|
|
|
if (HyperBase == NULL) {
|
|
MiDownPfnReferenceCount (HyperPhysicalPage, 2);
|
|
#if !defined (_X86PAE_)
|
|
MiDownPfnReferenceCount (RootPhysicalPage, 1);
|
|
MiUnmapSinglePage (PdeBase);
|
|
#else
|
|
for (i = 0; i < PD_PER_SYSTEM; i += 1) {
|
|
MiDownPfnReferenceCount (PageDirectoryFrames[i], 1);
|
|
}
|
|
MmUnmapLockedPages (PdeBase, MdlPageDirectory);
|
|
#endif
|
|
CurrentProcess->ForkInProgress = NULL;
|
|
UNLOCK_WS (CurrentProcess);
|
|
status = STATUS_INSUFFICIENT_RESOURCES;
|
|
goto ErrorReturn4;
|
|
}
|
|
#endif
|
|
|
|
|
|
// Initialize a page table MDL to lock and map the hyperspace page so it
|
|
// can be unlocked in the loop and the end of the loop without
|
|
// any testing to see if has a valid value the first time through.
|
|
|
|
|
|
#if defined (_WIN64)
|
|
MdlPage = RootPhysicalPage;
|
|
#else
|
|
MdlPage = HyperPhysicalPage;
|
|
#endif
|
|
|
|
NewPteMappedAddress = (PMMPTE)MiMapSinglePage (NULL,
|
|
MdlPage,
|
|
MmCached,
|
|
HighPagePriority);
|
|
|
|
if (NewPteMappedAddress == NULL) {
|
|
|
|
#if defined (_WIN64)
|
|
|
|
MiDownPfnReferenceCount (RootPhysicalPage, 3);
|
|
MiUnmapSinglePage (PpeBase);
|
|
MiUnmapSinglePage (PdeBase);
|
|
|
|
#else
|
|
MiDownPfnReferenceCount (HyperPhysicalPage, 2);
|
|
MiUnmapSinglePage (HyperBase);
|
|
#if !defined (_X86PAE_)
|
|
MiDownPfnReferenceCount (RootPhysicalPage, 1);
|
|
MiUnmapSinglePage (PdeBase);
|
|
#else
|
|
for (i = 0; i < PD_PER_SYSTEM; i += 1) {
|
|
MiDownPfnReferenceCount (PageDirectoryFrames[i], 1);
|
|
}
|
|
MmUnmapLockedPages (PdeBase, MdlPageDirectory);
|
|
#endif
|
|
|
|
#endif
|
|
|
|
CurrentProcess->ForkInProgress = NULL;
|
|
UNLOCK_WS (CurrentProcess);
|
|
status = STATUS_INSUFFICIENT_RESOURCES;
|
|
goto ErrorReturn4;
|
|
}
|
|
|
|
PointerNewPte = NewPteMappedAddress;
|
|
|
|
|
|
// Build new clone prototype PTE block and descriptor, note that
|
|
// each prototype PTE has a reference count following it.
|
|
|
|
|
|
ForkProtoPte = CloneProtos;
|
|
|
|
LockedForkPte = ForkProtoPte;
|
|
MiLockPagedAddress (LockedForkPte, FALSE);
|
|
|
|
CloneHeader->NumberOfPtes = (ULONG)NumberOfPrivatePages;
|
|
CloneHeader->NumberOfProcessReferences = 1;
|
|
CloneHeader->ClonePtes = CloneProtos;
|
|
|
|
|
|
|
|
CloneDescriptor->StartingVpn = (ULONG_PTR)CloneProtos;
|
|
CloneDescriptor->EndingVpn = (ULONG_PTR)((ULONG_PTR)CloneProtos +
|
|
NumberOfPrivatePages *
|
|
sizeof(MMCLONE_BLOCK));
|
|
CloneDescriptor->NumberOfReferences = 0;
|
|
CloneDescriptor->NumberOfPtes = (ULONG)NumberOfPrivatePages;
|
|
CloneDescriptor->CloneHeader = CloneHeader;
|
|
CloneDescriptor->PagedPoolQuotaCharge = sizeof(MMCLONE_BLOCK) *
|
|
NumberOfPrivatePages;
|
|
|
|
|
|
// Insert the clone descriptor for this fork operation into the
|
|
// process which was cloned.
|
|
|
|
|
|
MiInsertClone (CloneDescriptor);
|
|
|
|
|
|
// Examine each virtual address descriptor and create the
|
|
// proper structures for the new process.
|
|
|
|
|
|
Vad = MiGetFirstVad (CurrentProcess);
|
|
NewVad = FirstNewVad;
|
|
|
|
while (Vad != (PMMVAD)NULL) {
|
|
|
|
|
|
// Examine the VAD to determine its type and inheritance
|
|
// attribute.
|
|
|
|
|
|
if ((Vad->u.VadFlags.UserPhysicalPages == 0) &&
|
|
|
|
((Vad->u.VadFlags.PrivateMemory == 1) ||
|
|
(Vad->u2.VadFlags2.Inherit == MM_VIEW_SHARE))) {
|
|
|
|
|
|
// The virtual address descriptor should be shared in the
|
|
// forked process.
|
|
|
|
|
|
|
|
// Make a copy of the VAD for the new process, the new VADs
|
|
// are preallocated and linked together through the parent
|
|
// field.
|
|
|
|
|
|
NextVad = NewVad->Parent;
|
|
|
|
|
|
if (Vad->u.VadFlags.PrivateMemory == 1) {
|
|
*(PMMVAD_SHORT)NewVad = *(PMMVAD_SHORT)Vad;
|
|
NewVad->u.VadFlags.NoChange = 0;
|
|
} else {
|
|
*NewVad = *Vad;
|
|
}
|
|
|
|
if (NewVad->u.VadFlags.NoChange) {
|
|
if ((NewVad->u2.VadFlags2.OneSecured) ||
|
|
(NewVad->u2.VadFlags2.MultipleSecured)) {
|
|
|
|
|
|
// Eliminate these as the memory was secured
|
|
// only in this process, not in the new one.
|
|
|
|
|
|
NewVad->u2.VadFlags2.OneSecured = 0;
|
|
NewVad->u2.VadFlags2.MultipleSecured = 0;
|
|
NewVad->u2.VadFlags2.StoredInVad = 0;
|
|
NewVad->u3.List.Flink = NULL;
|
|
NewVad->u3.List.Blink = NULL;
|
|
}
|
|
if (NewVad->u2.VadFlags2.SecNoChange == 0) {
|
|
NewVad->u.VadFlags.NoChange = 0;
|
|
}
|
|
}
|
|
NewVad->Parent = NextVad;
|
|
|
|
|
|
// If the VAD refers to a section, up the view count for that
|
|
// section. This requires the PFN mutex to be held.
|
|
|
|
|
|
if ((Vad->u.VadFlags.PrivateMemory == 0) &&
|
|
(Vad->ControlArea != (PCONTROL_AREA)NULL)) {
|
|
|
|
|
|
// Increment the count of the number of views for the
|
|
// section object. This requires the PFN mutex to be held.
|
|
|
|
|
|
MiUpControlAreaRefs (Vad->ControlArea);
|
|
}
|
|
|
|
|
|
// Examine each PTE and create the appropriate PTE for the
|
|
// new process.
|
|
|
|
|
|
PointerPde = MiGetPdeAddress (MI_VPN_TO_VA (Vad->StartingVpn));
|
|
PointerPte = (volatile PMMPTE) MiGetPteAddress (
|
|
MI_VPN_TO_VA (Vad->StartingVpn));
|
|
LastPte = MiGetPteAddress (MI_VPN_TO_VA (Vad->EndingVpn));
|
|
FirstTime = TRUE;
|
|
|
|
while ((PMMPTE)PointerPte <= LastPte) {
|
|
|
|
|
|
// For each PTE contained in the VAD check the page table
|
|
// page, and if non-zero, make the appropriate modifications
|
|
// to copy the PTE to the new process.
|
|
|
|
|
|
if ((FirstTime) || MiIsPteOnPdeBoundary (PointerPte)) {
|
|
|
|
PointerPpe = MiGetPdeAddress (PointerPte);
|
|
PointerPde = MiGetPteAddress (PointerPte);
|
|
|
|
do {
|
|
|
|
while (!MiDoesPpeExistAndMakeValid (PointerPpe,
|
|
CurrentProcess,
|
|
FALSE,
|
|
&Waited)) {
|
|
|
|
|
|
// Page directory parent is empty, go to the next one.
|
|
|
|
|
|
PointerPpe += 1;
|
|
PointerPde = MiGetVirtualAddressMappedByPte (PointerPpe);
|
|
PointerPte = MiGetVirtualAddressMappedByPte (PointerPde);
|
|
|
|
if ((PMMPTE)PointerPte > LastPte) {
|
|
|
|
|
|
// All done with this VAD, exit loop.
|
|
|
|
|
|
goto AllDone;
|
|
}
|
|
}
|
|
|
|
Waited = 0;
|
|
|
|
while (!MiDoesPdeExistAndMakeValid (PointerPde,
|
|
CurrentProcess,
|
|
FALSE,
|
|
&Waited)) {
|
|
|
|
|
|
// This page directory is empty, go to the next one.
|
|
|
|
|
|
PointerPde += 1;
|
|
PointerPte = MiGetVirtualAddressMappedByPte (PointerPde);
|
|
|
|
if ((PMMPTE)PointerPte > LastPte) {
|
|
|
|
|
|
// All done with this VAD, exit loop.
|
|
|
|
|
|
goto AllDone;
|
|
}
|
|
#if defined (_WIN64)
|
|
if (MiIsPteOnPdeBoundary (PointerPde)) {
|
|
PointerPpe = MiGetPteAddress (PointerPde);
|
|
Waited = 1;
|
|
break;
|
|
}
|
|
#endif
|
|
}
|
|
|
|
} while (Waited != 0);
|
|
|
|
FirstTime = FALSE;
|
|
|
|
#if defined (_WIN64)
|
|
|
|
// Calculate the address of the PPE in the new process's
|
|
// page directory parent page.
|
|
|
|
|
|
PointerNewPpe = &PpeBase[MiGetPdeOffset(PointerPte)];
|
|
|
|
if (PointerNewPpe->u.Long == 0) {
|
|
|
|
|
|
// No physical page has been allocated yet, get a page
|
|
// and map it in as a transition page. This will
|
|
// become a page table page for the new process.
|
|
|
|
|
|
ReleasedWorkingSetMutex =
|
|
MiDoneWithThisPageGetAnother (&PageDirFrameIndex,
|
|
PointerPpe,
|
|
CurrentProcess);
|
|
|
|
MI_ZERO_USED_PAGETABLE_ENTRIES (MI_PFN_ELEMENT(PageDirFrameIndex));
|
|
|
|
if (ReleasedWorkingSetMutex) {
|
|
|
|
do {
|
|
|
|
MiDoesPpeExistAndMakeValid (PointerPpe,
|
|
CurrentProcess,
|
|
FALSE,
|
|
&Waited);
|
|
|
|
Waited = 0;
|
|
|
|
MiDoesPdeExistAndMakeValid (PointerPde,
|
|
CurrentProcess,
|
|
FALSE,
|
|
&Waited);
|
|
} while (Waited != 0);
|
|
}
|
|
|
|
|
|
// Hand initialize this PFN as normal initialization
|
|
// would do it for the process whose context we are
|
|
// attached to.
|
|
|
|
// The PFN lock must be held while initializing the
|
|
// frame to prevent those scanning the database for
|
|
// free frames from taking it after we fill in the
|
|
// u2 field.
|
|
|
|
|
|
MiBuildForkPageTable (PageDirFrameIndex,
|
|
PointerPpe,
|
|
PointerNewPpe,
|
|
RootPhysicalPage,
|
|
PfnPpPage);
|
|
|
|
|
|
// Map the new page directory page into the system
|
|
// portion of the address space. Note that hyperspace
|
|
// cannot be used as other operations (allocating
|
|
// nonpaged pool at DPC level) could cause the
|
|
// hyperspace page being used to be reused.
|
|
|
|
|
|
MiDownPfnReferenceCount (MdlDirPage, 1);
|
|
|
|
MdlDirPage = PageDirFrameIndex;
|
|
|
|
ASSERT (PdeBase != NULL);
|
|
|
|
PdeBase = (PMMPTE)MiMapSinglePage (PdeBase,
|
|
MdlDirPage,
|
|
MmCached,
|
|
HighPagePriority);
|
|
|
|
MiUpPfnReferenceCount (MdlDirPage, 1);
|
|
|
|
PointerNewPde = PdeBase;
|
|
}
|
|
else {
|
|
ASSERT (PointerNewPpe->u.Hard.Valid == 1 ||
|
|
PointerNewPpe->u.Soft.Transition == 1);
|
|
|
|
if (PointerNewPpe->u.Hard.Valid == 1) {
|
|
PageDirFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (PointerNewPpe);
|
|
}
|
|
else {
|
|
PageDirFrameIndex = MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (PointerNewPpe);
|
|
}
|
|
}
|
|
|
|
|
|
// Calculate the address of the new PDE to build.
|
|
// Note that FirstTime could be true, yet the page
|
|
// directory page might already be built.
|
|
|
|
|
|
PointerNewPde = (PMMPTE)((ULONG_PTR)PAGE_ALIGN(PointerNewPde) |
|
|
BYTE_OFFSET (PointerPde));
|
|
|
|
PdePhysicalPage = PageDirFrameIndex;
|
|
|
|
PfnPdPage = MI_PFN_ELEMENT (PdePhysicalPage);
|
|
|
|
UsedPageDirectoryEntries = (PVOID)PfnPdPage;
|
|
#endif
|
|
|
|
|
|
// Calculate the address of the PDE in the new process's
|
|
// page directory page.
|
|
|
|
|
|
PpePdeOffset = MiGetPpePdeOffset(MiGetVirtualAddressMappedByPte(PointerPte));
|
|
PointerNewPde = &PdeBase[PpePdeOffset];
|
|
|
|
if (PointerNewPde->u.Long == 0) {
|
|
|
|
|
|
// No physical page has been allocated yet, get a page
|
|
// and map it in as a transition page. This will
|
|
// become a page table page for the new process.
|
|
|
|
|
|
ReleasedWorkingSetMutex =
|
|
MiDoneWithThisPageGetAnother (&PageFrameIndex,
|
|
PointerPde,
|
|
CurrentProcess);
|
|
|
|
if (ReleasedWorkingSetMutex) {
|
|
|
|
do {
|
|
|
|
MiDoesPpeExistAndMakeValid (PointerPpe,
|
|
CurrentProcess,
|
|
FALSE,
|
|
&Waited);
|
|
|
|
Waited = 0;
|
|
|
|
MiDoesPdeExistAndMakeValid (PointerPde,
|
|
CurrentProcess,
|
|
FALSE,
|
|
&Waited);
|
|
} while (Waited != 0);
|
|
}
|
|
|
|
|
|
// Hand initialize this PFN as normal initialization
|
|
// would do it for the process whose context we are
|
|
// attached to.
|
|
|
|
// The PFN lock must be held while initializing the
|
|
// frame to prevent those scanning the database for
|
|
// free frames from taking it after we fill in the
|
|
// u2 field.
|
|
|
|
|
|
#if defined (_X86PAE_)
|
|
PdePhysicalPage = PageDirectoryFrames[MiGetPdPteOffset(MiGetVirtualAddressMappedByPte(PointerPte))];
|
|
PfnPdPage = MI_PFN_ELEMENT (PdePhysicalPage);
|
|
#endif
|
|
|
|
MiBuildForkPageTable (PageFrameIndex,
|
|
PointerPde,
|
|
PointerNewPde,
|
|
PdePhysicalPage,
|
|
PfnPdPage);
|
|
|
|
#if defined (_WIN64)
|
|
MI_INCREMENT_USED_PTES_BY_HANDLE (UsedPageDirectoryEntries);
|
|
#endif
|
|
|
|
|
|
// Map the new page table page into the system portion
|
|
// of the address space. Note that hyperspace
|
|
// cannot be used as other operations (allocating
|
|
// nonpaged pool at DPC level) could cause the
|
|
// hyperspace page being used to be reused.
|
|
|
|
|
|
ASSERT (NewPteMappedAddress != NULL);
|
|
|
|
MiDownPfnReferenceCount (MdlPage, 1);
|
|
|
|
MdlPage = PageFrameIndex;
|
|
|
|
PointerNewPte = (PMMPTE)MiMapSinglePage (NewPteMappedAddress,
|
|
MdlPage,
|
|
MmCached,
|
|
HighPagePriority);
|
|
|
|
ASSERT (PointerNewPte != NULL);
|
|
|
|
MiUpPfnReferenceCount (MdlPage, 1);
|
|
}
|
|
|
|
|
|
// Calculate the address of the new PTE to build.
|
|
// Note that FirstTime could be true, yet the page
|
|
// table page already built.
|
|
|
|
|
|
PointerNewPte = (PMMPTE)((ULONG_PTR)PAGE_ALIGN(PointerNewPte) |
|
|
BYTE_OFFSET (PointerPte));
|
|
|
|
#ifdef _WIN64
|
|
UsedPageTableEntries = (PVOID)MI_PFN_ELEMENT((PFN_NUMBER)PointerNewPde->u.Hard.PageFrameNumber);
|
|
#else
|
|
#if !defined (_X86PAE_)
|
|
UsedPageTableEntries = (PVOID)&HyperBase->UsedPageTableEntries
|
|
[MiGetPteOffset( PointerPte )];
|
|
#else
|
|
UsedPageTableEntries = (PVOID)&HyperBase->UsedPageTableEntries
|
|
[MiGetPpePdeOffset(MiGetVirtualAddressMappedByPte(PointerPte))];
|
|
#endif
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
// Make the fork prototype PTE location resident.
|
|
|
|
|
|
if (PAGE_ALIGN (ForkProtoPte) != PAGE_ALIGN (LockedForkPte)) {
|
|
MiUnlockPagedAddress (LockedForkPte, FALSE);
|
|
LockedForkPte = ForkProtoPte;
|
|
MiLockPagedAddress (LockedForkPte, FALSE);
|
|
}
|
|
|
|
MiMakeSystemAddressValid (PointerPte, CurrentProcess);
|
|
|
|
PteContents = *PointerPte;
|
|
|
|
|
|
// Check each PTE.
|
|
|
|
|
|
if (PteContents.u.Long == 0) {
|
|
NOTHING;
|
|
|
|
} else if (PteContents.u.Hard.Valid == 1) {
|
|
|
|
|
|
// Valid.
|
|
|
|
|
|
Pfn2 = MI_PFN_ELEMENT (PteContents.u.Hard.PageFrameNumber);
|
|
VirtualAddress = MiGetVirtualAddressMappedByPte (PointerPte);
|
|
WorkingSetIndex = MiLocateWsle (VirtualAddress,
|
|
MmWorkingSetList,
|
|
Pfn2->u1.WsIndex);
|
|
|
|
ASSERT (WorkingSetIndex != WSLE_NULL_INDEX);
|
|
|
|
if (Pfn2->u3.e1.PrototypePte == 1) {
|
|
|
|
|
|
// This PTE is already in prototype PTE format.
|
|
|
|
|
|
|
|
// This is a prototype PTE. The PFN database does
|
|
// not contain the contents of this PTE it contains
|
|
// the contents of the prototype PTE. This PTE must
|
|
// be reconstructed to contain a pointer to the
|
|
// prototype PTE.
|
|
|
|
// The working set list entry contains information about
|
|
// how to reconstruct the PTE.
|
|
|
|
|
|
if (MmWsle[WorkingSetIndex].u1.e1.SameProtectAsProto
|
|
== 0) {
|
|
|
|
|
|
// The protection for the prototype PTE is in the
|
|
// WSLE.
|
|
|
|
|
|
TempPte.u.Long = 0;
|
|
TempPte.u.Soft.Protection =
|
|
MI_GET_PROTECTION_FROM_WSLE(&MmWsle[WorkingSetIndex]);
|
|
TempPte.u.Soft.PageFileHigh = MI_PTE_LOOKUP_NEEDED;
|
|
|
|
} else {
|
|
|
|
|
|
// The protection is in the prototype PTE.
|
|
|
|
|
|
TempPte.u.Long = MiProtoAddressForPte (
|
|
Pfn2->PteAddress);
|
|
// TempPte.u.Proto.ForkType =
|
|
// MmWsle[WorkingSetIndex].u1.e1.ForkType;
|
|
}
|
|
|
|
TempPte.u.Proto.Prototype = 1;
|
|
MI_WRITE_INVALID_PTE (PointerNewPte, TempPte);
|
|
|
|
|
|
// A PTE is now non-zero, increment the used page
|
|
// table entries counter.
|
|
|
|
|
|
MI_INCREMENT_USED_PTES_BY_HANDLE (UsedPageTableEntries);
|
|
|
|
|
|
// Check to see if this is a fork prototype PTE,
|
|
// and if it is increment the reference count
|
|
// which is in the longword following the PTE.
|
|
|
|
|
|
if (MiLocateCloneAddress ((PVOID)Pfn2->PteAddress) !=
|
|
(PMMCLONE_DESCRIPTOR)NULL) {
|
|
|
|
|
|
// The reference count field, or the prototype PTE
|
|
// for that matter may not be in the working set.
|
|
|
|
|
|
CloneProto = (PMMCLONE_BLOCK)Pfn2->PteAddress;
|
|
|
|
MiUpCloneProtoRefCount (CloneProto,
|
|
CurrentProcess);
|
|
|
|
if (PAGE_ALIGN (ForkProtoPte) !=
|
|
PAGE_ALIGN (LockedForkPte)) {
|
|
MiUnlockPagedAddress (LockedForkPte, FALSE);
|
|
LockedForkPte = ForkProtoPte;
|
|
MiLockPagedAddress (LockedForkPte, FALSE);
|
|
}
|
|
|
|
MiMakeSystemAddressValid (PointerPte,
|
|
CurrentProcess);
|
|
}
|
|
|
|
} else {
|
|
|
|
|
|
// This is a private page, create a fork prototype PTE
|
|
// which becomes the "prototype" PTE for this page.
|
|
// The protection is the same as that in the prototype'
|
|
// PTE so the WSLE does not need to be updated.
|
|
|
|
|
|
MI_MAKE_VALID_PTE_WRITE_COPY (PointerPte);
|
|
|
|
ForkProtoPte->ProtoPte = *PointerPte;
|
|
ForkProtoPte->CloneRefCount = 2;
|
|
|
|
|
|
// Transform the PFN element to reference this new fork
|
|
// prototype PTE.
|
|
|
|
|
|
Pfn2->PteAddress = &ForkProtoPte->ProtoPte;
|
|
Pfn2->u3.e1.PrototypePte = 1;
|
|
|
|
ContainingPte = MiGetPteAddress(&ForkProtoPte->ProtoPte);
|
|
if (ContainingPte->u.Hard.Valid == 0) {
|
|
#if !defined (_WIN64)
|
|
if (!NT_SUCCESS(MiCheckPdeForPagedPool (&ForkProtoPte->ProtoPte))) {
|
|
#endif
|
|
KeBugCheckEx (MEMORY_MANAGEMENT,
|
|
0x61940,
|
|
(ULONG_PTR)&ForkProtoPte->ProtoPte,
|
|
(ULONG_PTR)ContainingPte->u.Long,
|
|
(ULONG_PTR)MiGetVirtualAddressMappedByPte(&ForkProtoPte->ProtoPte));
|
|
#if !defined (_WIN64)
|
|
}
|
|
#endif
|
|
}
|
|
Pfn2->PteFrame = MI_GET_PAGE_FRAME_FROM_PTE (ContainingPte);
|
|
|
|
|
|
|
|
// Increment the share count for the page containing the
|
|
// fork prototype PTEs as we have just placed a valid
|
|
// PTE into the page.
|
|
|
|
|
|
PfnForkPtePage = MI_PFN_ELEMENT (
|
|
ContainingPte->u.Hard.PageFrameNumber );
|
|
|
|
MiUpForkPageShareCount (PfnForkPtePage);
|
|
|
|
|
|
// Change the protection in the PFN database to COPY
|
|
// on write, if writable.
|
|
|
|
|
|
MI_MAKE_PROTECT_WRITE_COPY (Pfn2->OriginalPte);
|
|
|
|
|
|
// Put the protection into the WSLE and mark the WSLE
|
|
// to indicate that the protection field for the PTE
|
|
// is the same as the prototype PTE.
|
|
|
|
|
|
MmWsle[WorkingSetIndex].u1.e1.Protection =
|
|
MI_GET_PROTECTION_FROM_SOFT_PTE(&Pfn2->OriginalPte);
|
|
|
|
MmWsle[WorkingSetIndex].u1.e1.SameProtectAsProto = 1;
|
|
|
|
TempPte.u.Long = MiProtoAddressForPte (Pfn2->PteAddress);
|
|
TempPte.u.Proto.Prototype = 1;
|
|
MI_WRITE_INVALID_PTE (PointerNewPte, TempPte);
|
|
|
|
|
|
// A PTE is now non-zero, increment the used page
|
|
// table entries counter.
|
|
|
|
|
|
MI_INCREMENT_USED_PTES_BY_HANDLE (UsedPageTableEntries);
|
|
|
|
|
|
// One less private page (it's now shared).
|
|
|
|
|
|
CurrentProcess->NumberOfPrivatePages -= 1;
|
|
|
|
ForkProtoPte += 1;
|
|
NumberOfForkPtes += 1;
|
|
|
|
}
|
|
|
|
} else if (PteContents.u.Soft.Prototype == 1) {
|
|
|
|
|
|
// Prototype PTE, check to see if this is a fork
|
|
// prototype PTE already. Note that if COW is set,
|
|
// the PTE can just be copied (fork compatible format).
|
|
|
|
|
|
MI_WRITE_INVALID_PTE (PointerNewPte, PteContents);
|
|
|
|
|
|
// A PTE is now non-zero, increment the used page
|
|
// table entries counter.
|
|
|
|
|
|
MI_INCREMENT_USED_PTES_BY_HANDLE (UsedPageTableEntries);
|
|
|
|
|
|
// Check to see if this is a fork prototype PTE,
|
|
// and if it is increment the reference count
|
|
// which is in the longword following the PTE.
|
|
|
|
|
|
CloneProto = (PMMCLONE_BLOCK)(MiPteToProto(PointerPte));
|
|
|
|
if (MiLocateCloneAddress ((PVOID)CloneProto) !=
|
|
(PMMCLONE_DESCRIPTOR)NULL) {
|
|
|
|
|
|
// The reference count field, or the prototype PTE
|
|
// for that matter may not be in the working set.
|
|
|
|
|
|
MiUpCloneProtoRefCount (CloneProto,
|
|
CurrentProcess);
|
|
|
|
if (PAGE_ALIGN (ForkProtoPte) !=
|
|
PAGE_ALIGN (LockedForkPte)) {
|
|
MiUnlockPagedAddress (LockedForkPte, FALSE);
|
|
LockedForkPte = ForkProtoPte;
|
|
MiLockPagedAddress (LockedForkPte, FALSE);
|
|
}
|
|
|
|
MiMakeSystemAddressValid (PointerPte,
|
|
CurrentProcess);
|
|
}
|
|
|
|
} else if (PteContents.u.Soft.Transition == 1) {
|
|
|
|
|
|
// Transition.
|
|
|
|
|
|
if (MiHandleForkTransitionPte (PointerPte,
|
|
PointerNewPte,
|
|
ForkProtoPte)) {
|
|
|
|
// PTE is no longer transition, try again.
|
|
|
|
|
|
continue;
|
|
}
|
|
|
|
|
|
// A PTE is now non-zero, increment the used page
|
|
// table entries counter.
|
|
|
|
|
|
MI_INCREMENT_USED_PTES_BY_HANDLE (UsedPageTableEntries);
|
|
|
|
|
|
// One less private page (it's now shared).
|
|
|
|
|
|
CurrentProcess->NumberOfPrivatePages -= 1;
|
|
|
|
ForkProtoPte += 1;
|
|
NumberOfForkPtes += 1;
|
|
|
|
} else {
|
|
|
|
|
|
// Page file format (may be demand zero).
|
|
|
|
|
|
if (IS_PTE_NOT_DEMAND_ZERO (PteContents)) {
|
|
|
|
if (PteContents.u.Soft.Protection == MM_DECOMMIT) {
|
|
|
|
|
|
// This is a decommitted PTE, just move it
|
|
// over to the new process. Don't increment
|
|
// the count of private pages.
|
|
|
|
|
|
MI_WRITE_INVALID_PTE (PointerNewPte, PteContents);
|
|
} else {
|
|
|
|
|
|
// The PTE is not demand zero, move the PTE to
|
|
// a fork prototype PTE and make this PTE and
|
|
// the new processes PTE refer to the fork
|
|
// prototype PTE.
|
|
|
|
|
|
ForkProtoPte->ProtoPte = PteContents;
|
|
|
|
|
|
// Make the protection write-copy if writable.
|
|
|
|
|
|
MI_MAKE_PROTECT_WRITE_COPY (ForkProtoPte->ProtoPte);
|
|
|
|
ForkProtoPte->CloneRefCount = 2;
|
|
|
|
TempPte.u.Long =
|
|
MiProtoAddressForPte (&ForkProtoPte->ProtoPte);
|
|
|
|
TempPte.u.Proto.Prototype = 1;
|
|
|
|
MI_WRITE_INVALID_PTE (PointerPte, TempPte);
|
|
MI_WRITE_INVALID_PTE (PointerNewPte, TempPte);
|
|
|
|
|
|
// One less private page (it's now shared).
|
|
|
|
|
|
CurrentProcess->NumberOfPrivatePages -= 1;
|
|
|
|
ForkProtoPte += 1;
|
|
NumberOfForkPtes += 1;
|
|
}
|
|
} else {
|
|
|
|
|
|
// The page is demand zero, make the new process's
|
|
// page demand zero.
|
|
|
|
|
|
MI_WRITE_INVALID_PTE (PointerNewPte, PteContents);
|
|
}
|
|
|
|
|
|
// A PTE is now non-zero, increment the used page
|
|
// table entries counter.
|
|
|
|
|
|
MI_INCREMENT_USED_PTES_BY_HANDLE (UsedPageTableEntries);
|
|
}
|
|
|
|
PointerPte += 1;
|
|
PointerNewPte += 1;
|
|
|
|
} // end while for PTEs
|
|
AllDone:
|
|
NewVad = NewVad->Parent;
|
|
}
|
|
Vad = MiGetNextVad (Vad);
|
|
|
|
} // end while for VADs
|
|
|
|
|
|
// Unlock paged pool page.
|
|
|
|
|
|
MiUnlockPagedAddress (LockedForkPte, FALSE);
|
|
|
|
|
|
// Unmap the PD Page and hyper space page.
|
|
|
|
|
|
#if defined (_WIN64)
|
|
MiUnmapSinglePage (PpeBase);
|
|
#endif
|
|
|
|
#if !defined (_X86PAE_)
|
|
MiUnmapSinglePage (PdeBase);
|
|
#else
|
|
MmUnmapLockedPages (PdeBase, MdlPageDirectory);
|
|
#endif
|
|
|
|
#if !defined (_WIN64)
|
|
MiUnmapSinglePage (HyperBase);
|
|
#endif
|
|
|
|
MiUnmapSinglePage (NewPteMappedAddress);
|
|
|
|
#if defined (_WIN64)
|
|
MiDownPfnReferenceCount (RootPhysicalPage, 1);
|
|
#endif
|
|
|
|
#if defined (_X86PAE_)
|
|
for (i = 0; i < PD_PER_SYSTEM; i += 1) {
|
|
MiDownPfnReferenceCount (PageDirectoryFrames[i], 1);
|
|
}
|
|
#else
|
|
MiDownPfnReferenceCount (MdlDirPage, 1);
|
|
#endif
|
|
|
|
#if !defined (_WIN64)
|
|
MiDownPfnReferenceCount (HyperPhysicalPage, 1);
|
|
#endif
|
|
|
|
MiDownPfnReferenceCount (MdlPage, 1);
|
|
|
|
|
|
// Make the count of private pages match between the two processes.
|
|
|
|
|
|
ASSERT ((SPFN_NUMBER)CurrentProcess->NumberOfPrivatePages >= 0);
|
|
|
|
ProcessToInitialize->NumberOfPrivatePages =
|
|
CurrentProcess->NumberOfPrivatePages;
|
|
|
|
ASSERT (NumberOfForkPtes <= CloneDescriptor->NumberOfPtes);
|
|
|
|
if (NumberOfForkPtes != 0) {
|
|
|
|
|
|
// The number of fork PTEs is non-zero, set the values
|
|
// into the structures.
|
|
|
|
|
|
CloneHeader->NumberOfPtes = NumberOfForkPtes;
|
|
CloneDescriptor->NumberOfReferences = NumberOfForkPtes;
|
|
CloneDescriptor->NumberOfPtes = NumberOfForkPtes;
|
|
|
|
} else {
|
|
|
|
|
|
// There were no fork PTEs created. Remove the clone descriptor
|
|
// from this process and clean up the related structures.
|
|
// Note - must be holding the working set mutex and not holding
|
|
// the PFN lock.
|
|
|
|
|
|
MiRemoveClone (CloneDescriptor);
|
|
|
|
UNLOCK_WS (CurrentProcess);
|
|
|
|
ExFreePool (CloneDescriptor->CloneHeader->ClonePtes);
|
|
|
|
ExFreePool (CloneDescriptor->CloneHeader);
|
|
|
|
|
|
// Return the pool for the global structures referenced by the
|
|
// clone descriptor.
|
|
|
|
|
|
PsReturnPoolQuota (CurrentProcess,
|
|
PagedPool,
|
|
CloneDescriptor->PagedPoolQuotaCharge);
|
|
|
|
PsReturnPoolQuota (CurrentProcess, NonPagedPool, sizeof(MMCLONE_HEADER));
|
|
|
|
ExFreePool (CloneDescriptor);
|
|
|
|
LOCK_WS (CurrentProcess);
|
|
}
|
|
|
|
MiDownShareCountFlushEntireTb (PageFrameIndex);
|
|
|
|
#if defined (_WIN64)
|
|
MiDownShareCountFlushEntireTb (PageDirFrameIndex);
|
|
#endif
|
|
|
|
PageFrameIndex = (PFN_NUMBER)-1;
|
|
PageDirFrameIndex = (PFN_NUMBER)-1;
|
|
|
|
|
|
// Copy the clone descriptors from this process to the new process.
|
|
|
|
|
|
Clone = MiGetFirstClone ();
|
|
CloneList = &FirstNewClone;
|
|
CloneFailed = FALSE;
|
|
|
|
while (Clone != (PMMCLONE_DESCRIPTOR)NULL) {
|
|
|
|
|
|
// Increment the count of processes referencing this clone block.
|
|
|
|
|
|
MiUpCloneProcessRefCount (Clone);
|
|
|
|
do {
|
|
NewClone = ExAllocatePoolWithTag (NonPagedPool,
|
|
sizeof( MMCLONE_DESCRIPTOR),
|
|
' mM');
|
|
|
|
if (NewClone != NULL) {
|
|
break;
|
|
}
|
|
|
|
|
|
// There are insufficient resources to continue this operation,
|
|
// however, to properly clean up at this point, all the
|
|
// clone headers must be allocated, so when the cloned process
|
|
// is deleted, the clone headers will be found. if the pool
|
|
// is not readily available, loop periodically trying for it.
|
|
// Force the clone operation to fail so the pool will soon be
|
|
// released.
|
|
|
|
|
|
CloneFailed = TRUE;
|
|
status = STATUS_INSUFFICIENT_RESOURCES;
|
|
|
|
KeDelayExecutionThread (KernelMode,
|
|
FALSE,
|
|
(PLARGE_INTEGER)&MmShortTime);
|
|
continue;
|
|
} while (TRUE);
|
|
|
|
*NewClone = *Clone;
|
|
|
|
*CloneList = NewClone;
|
|
CloneList = &NewClone->Parent;
|
|
Clone = MiGetNextClone (Clone);
|
|
}
|
|
|
|
*CloneList = (PMMCLONE_DESCRIPTOR)NULL;
|
|
|
|
|
|
// Release the working set mutex and the address creation mutex from
|
|
// the current process as all the necessary information is now
|
|
// captured.
|
|
|
|
|
|
UNLOCK_WS (CurrentProcess);
|
|
|
|
CurrentProcess->ForkInProgress = NULL;
|
|
|
|
UNLOCK_ADDRESS_SPACE (CurrentProcess);
|
|
|
|
|
|
// As we have updated many PTEs to clear dirty bits, flush the
|
|
// TB cache. Note that this was not done every time we changed
|
|
// a valid PTE so other threads could be modifying the address
|
|
// space without causing copy on writes. (Too bad).
|
|
|
|
|
|
|
|
|
|
// Attach to the process to initialize and insert the vad and clone
|
|
// descriptors into the tree.
|
|
|
|
|
|
if (Attached) {
|
|
KeDetachProcess ();
|
|
Attached = FALSE;
|
|
}
|
|
|
|
if (PsGetCurrentProcess() != ProcessToInitialize) {
|
|
Attached = TRUE;
|
|
KeAttachProcess (&ProcessToInitialize->Pcb);
|
|
}
|
|
|
|
CurrentProcess = ProcessToInitialize;
|
|
|
|
|
|
// We are now in the context of the new process, build the
|
|
// VAD list and the clone list.
|
|
|
|
|
|
Vad = FirstNewVad;
|
|
VadInsertFailed = FALSE;
|
|
|
|
LOCK_WS (CurrentProcess);
|
|
|
|
while (Vad != (PMMVAD)NULL) {
|
|
|
|
NextVad = Vad->Parent;
|
|
|
|
try {
|
|
|
|
if (VadInsertFailed) {
|
|
Vad->u.VadFlags.CommitCharge = MM_MAX_COMMIT;
|
|
}
|
|
|
|
MiInsertVad (Vad);
|
|
|
|
} except (EXCEPTION_EXECUTE_HANDLER) {
|
|
|
|
|
|
// Charging quota for the VAD failed, set the
|
|
// remaining quota fields in this VAD and all
|
|
// subsequent VADs to zero so the VADs can be
|
|
// inserted and later deleted.
|
|
|
|
|
|
VadInsertFailed = TRUE;
|
|
status = GetExceptionCode();
|
|
|
|
|
|
// Do the loop again for this VAD.
|
|
|
|
|
|
continue;
|
|
}
|
|
|
|
|
|
// Update the current virtual size.
|
|
|
|
|
|
CurrentProcess->VirtualSize += PAGE_SIZE +
|
|
((Vad->EndingVpn - Vad->StartingVpn) >> PAGE_SHIFT);
|
|
|
|
Vad = NextVad;
|
|
}
|
|
|
|
UNLOCK_WS (CurrentProcess);
|
|
//MmUnlockCode (MiCloneProcessAddressSpace, 5000);
|
|
|
|
|
|
// Update the peak virtual size.
|
|
|
|
|
|
CurrentProcess->PeakVirtualSize = CurrentProcess->VirtualSize;
|
|
|
|
Clone = FirstNewClone;
|
|
TotalPagedPoolCharge = 0;
|
|
TotalNonPagedPoolCharge = 0;
|
|
|
|
while (Clone != (PMMCLONE_DESCRIPTOR)NULL) {
|
|
|
|
NextClone = Clone->Parent;
|
|
MiInsertClone (Clone);
|
|
|
|
|
|
// Calculate the page pool and non-paged pool to charge for these
|
|
// operations.
|
|
|
|
|
|
TotalPagedPoolCharge += Clone->PagedPoolQuotaCharge;
|
|
TotalNonPagedPoolCharge += sizeof(MMCLONE_HEADER);
|
|
|
|
Clone = NextClone;
|
|
}
|
|
|
|
if (CloneFailed || VadInsertFailed) {
|
|
|
|
CurrentProcess->ForkWasSuccessful = MM_FORK_FAILED;
|
|
|
|
if (Attached) {
|
|
KeDetachProcess ();
|
|
}
|
|
KdPrint(("MMFORK: vad insert failed\n"));
|
|
|
|
return status;
|
|
}
|
|
|
|
try {
|
|
|
|
PageTablePage = 1;
|
|
PsChargePoolQuota (CurrentProcess, PagedPool, TotalPagedPoolCharge);
|
|
PageTablePage = 0;
|
|
PsChargePoolQuota (CurrentProcess, NonPagedPool, TotalNonPagedPoolCharge);
|
|
|
|
} except (EXCEPTION_EXECUTE_HANDLER) {
|
|
|
|
if (PageTablePage == 0) {
|
|
PsReturnPoolQuota (CurrentProcess, PagedPool, TotalPagedPoolCharge);
|
|
}
|
|
KdPrint(("MMFORK: pool quota failed\n"));
|
|
|
|
CurrentProcess->ForkWasSuccessful = MM_FORK_FAILED;
|
|
|
|
if (Attached) {
|
|
KeDetachProcess ();
|
|
}
|
|
return GetExceptionCode();
|
|
}
|
|
|
|
ASSERT (ProcessToClone->ForkWasSuccessful == MM_FORK_SUCCEEDED);
|
|
ASSERT (CurrentProcess->ForkWasSuccessful == MM_FORK_SUCCEEDED);
|
|
|
|
if (Attached) {
|
|
KeDetachProcess ();
|
|
}
|
|
|
|
#if DBG
|
|
if (MmDebug & MM_DBG_FORK) {
|
|
DbgPrint("ending clone operation process to clone = %lx\n",
|
|
ProcessToClone);
|
|
}
|
|
#endif //DBG
|
|
|
|
return STATUS_SUCCESS;
|
|
|
|
|
|
// Error returns.
|
|
|
|
|
|
ErrorReturn4:
|
|
if (PageTablePage == 2) {
|
|
NOTHING;
|
|
}
|
|
else if (PageTablePage == 1) {
|
|
PsReturnPoolQuota (CurrentProcess, PagedPool, sizeof(MMCLONE_BLOCK) *
|
|
NumberOfPrivatePages);
|
|
}
|
|
else {
|
|
ASSERT (PageTablePage == 0);
|
|
PsReturnPoolQuota (CurrentProcess, PagedPool, sizeof(MMCLONE_BLOCK) *
|
|
NumberOfPrivatePages);
|
|
PsReturnPoolQuota (CurrentProcess, NonPagedPool, sizeof(MMCLONE_HEADER));
|
|
}
|
|
|
|
NewVad = FirstNewVad;
|
|
while (NewVad != NULL) {
|
|
Vad = NewVad->Parent;
|
|
ExFreePool (NewVad);
|
|
NewVad = Vad;
|
|
}
|
|
|
|
ExFreePool (CloneDescriptor);
|
|
ErrorReturn3:
|
|
ExFreePool (CloneHeader);
|
|
ErrorReturn2:
|
|
ExFreePool (CloneProtos);
|
|
ErrorReturn1:
|
|
UNLOCK_ADDRESS_SPACE (CurrentProcess);
|
|
ASSERT (CurrentProcess->ForkWasSuccessful == MM_FORK_SUCCEEDED);
|
|
if (Attached) {
|
|
KeDetachProcess ();
|
|
}
|
|
return status;
|
|
}
|
|
|
|
ULONG
|
|
MiDecrementCloneBlockReference (
|
|
IN PMMCLONE_DESCRIPTOR CloneDescriptor,
|
|
IN PMMCLONE_BLOCK CloneBlock,
|
|
IN PEPROCESS CurrentProcess
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine decrements the reference count field of a "fork prototype
|
|
PTE" (clone-block). If the reference count becomes zero, the reference
|
|
count for the clone-descriptor is decremented and if that becomes zero,
|
|
it is deallocated and the number of process count for the clone header is
|
|
decremented. If the number of process count becomes zero, the clone
|
|
header is deallocated.
|
|
|
|
Arguments:
|
|
|
|
CloneDescriptor - Supplies the clone descriptor which describes the
|
|
clone block.
|
|
|
|
CloneBlock - Supplies the clone block to decrement the reference count of.
|
|
|
|
CurrentProcess - Supplies the current process.
|
|
|
|
Return Value:
|
|
|
|
TRUE if the working set mutex was released, FALSE if it was not.
|
|
|
|
Environment:
|
|
|
|
Kernel mode, APCs disabled, working set mutex and PFN mutex held.
|
|
|
|
--*/
|
|
|
|
{
|
|
|
|
ULONG MutexReleased;
|
|
MMPTE CloneContents;
|
|
PMMPFN Pfn3;
|
|
KIRQL OldIrql;
|
|
LONG NewCount;
|
|
LOGICAL WsHeldSafe;
|
|
|
|
MutexReleased = FALSE;
|
|
OldIrql = APC_LEVEL;
|
|
|
|
MutexReleased = MiMakeSystemAddressValidPfnWs (CloneBlock, CurrentProcess);
|
|
|
|
while (CurrentProcess->ForkInProgress) {
|
|
MiWaitForForkToComplete (CurrentProcess);
|
|
MiMakeSystemAddressValidPfnWs (CloneBlock, CurrentProcess);
|
|
MutexReleased = TRUE;
|
|
}
|
|
|
|
CloneBlock->CloneRefCount -= 1;
|
|
NewCount = CloneBlock->CloneRefCount;
|
|
|
|
ASSERT (NewCount >= 0);
|
|
|
|
if (NewCount == 0) {
|
|
CloneContents = CloneBlock->ProtoPte;
|
|
} else {
|
|
CloneContents = ZeroPte;
|
|
}
|
|
|
|
if ((NewCount == 0) && (CloneContents.u.Long != 0)) {
|
|
|
|
|
|
// The last reference to a fork prototype PTE
|
|
// has been removed. Deallocate any page file
|
|
// space and the transition page, if any.
|
|
|
|
|
|
|
|
|
|
// Assert that the page is no longer valid.
|
|
|
|
|
|
ASSERT (CloneContents.u.Hard.Valid == 0);
|
|
|
|
|
|
// Assert that the PTE is not in subsection format (doesn't point
|
|
// to a file).
|
|
|
|
|
|
ASSERT (CloneContents.u.Soft.Prototype == 0);
|
|
|
|
if (CloneContents.u.Soft.Transition == 1) {
|
|
|
|
|
|
// Prototype PTE in transition, put the page
|
|
// on the free list.
|
|
|
|
|
|
Pfn3 = MI_PFN_ELEMENT (CloneContents.u.Trans.PageFrameNumber);
|
|
MI_SET_PFN_DELETED (Pfn3);
|
|
|
|
MiDecrementShareCount (Pfn3->PteFrame);
|
|
|
|
|
|
// Check the reference count for the page, if the reference
|
|
// count is zero and the page is not on the freelist,
|
|
// move the page to the free list, if the reference
|
|
// count is not zero, ignore this page.
|
|
// When the reference count goes to zero, it will be placed on the
|
|
// free list.
|
|
|
|
|
|
if ((Pfn3->u3.e2.ReferenceCount == 0) &&
|
|
(Pfn3->u3.e1.PageLocation != FreePageList)) {
|
|
|
|
MiUnlinkPageFromList (Pfn3);
|
|
MiReleasePageFileSpace (Pfn3->OriginalPte);
|
|
MiInsertPageInList (MmPageLocationList[FreePageList],
|
|
MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE(&CloneContents));
|
|
}
|
|
} else {
|
|
|
|
if (IS_PTE_NOT_DEMAND_ZERO (CloneContents)) {
|
|
MiReleasePageFileSpace (CloneContents);
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
// Decrement the number of references to the
|
|
// clone descriptor.
|
|
|
|
|
|
CloneDescriptor->NumberOfReferences -= 1;
|
|
|
|
if (CloneDescriptor->NumberOfReferences == 0) {
|
|
|
|
|
|
// There are no longer any PTEs in this process which refer
|
|
// to the fork prototype PTEs for this clone descriptor.
|
|
// Remove the CloneDescriptor and decrement the CloneHeader
|
|
// number of process's reference count.
|
|
|
|
|
|
CloneDescriptor->CloneHeader->NumberOfProcessReferences -= 1;
|
|
|
|
MiRemoveClone (CloneDescriptor);
|
|
|
|
if (CloneDescriptor->CloneHeader->NumberOfProcessReferences == 0) {
|
|
|
|
|
|
// There are no more processes pointing to this fork header
|
|
// blow it away.
|
|
|
|
|
|
UNLOCK_PFN (OldIrql);
|
|
|
|
|
|
// The working set lock may have been acquired safely or unsafely
|
|
// by our caller. Handle both cases here and below.
|
|
|
|
|
|
UNLOCK_WS_REGARDLESS (CurrentProcess, WsHeldSafe);
|
|
|
|
MutexReleased = TRUE;
|
|
|
|
|
|
#if DBG
|
|
{
|
|
|
|
ULONG i;
|
|
PMMCLONE_BLOCK OldCloneBlock;
|
|
|
|
OldCloneBlock = CloneDescriptor->CloneHeader->ClonePtes;
|
|
for (i = 0; i < CloneDescriptor->CloneHeader->NumberOfPtes; i += 1) {
|
|
if (OldCloneBlock->CloneRefCount != 0) {
|
|
DbgPrint("fork block with non zero ref count %lx %lx %lx\n",
|
|
OldCloneBlock, CloneDescriptor,
|
|
CloneDescriptor->CloneHeader);
|
|
KeBugCheckEx (MEMORY_MANAGEMENT, 1, 0, 0, 0);
|
|
}
|
|
}
|
|
|
|
if (MmDebug & MM_DBG_FORK) {
|
|
DbgPrint("removing clone header at address %lx\n",
|
|
CloneDescriptor->CloneHeader);
|
|
}
|
|
|
|
}
|
|
#endif //DBG
|
|
|
|
ExFreePool (CloneDescriptor->CloneHeader->ClonePtes);
|
|
|
|
ExFreePool (CloneDescriptor->CloneHeader);
|
|
|
|
|
|
// The working set lock may have been acquired safely or unsafely
|
|
// by our caller. Reacquire it in the same manner our caller did.
|
|
|
|
|
|
LOCK_WS_REGARDLESS (CurrentProcess, WsHeldSafe);
|
|
|
|
LOCK_PFN (OldIrql);
|
|
}
|
|
|
|
#if DBG
|
|
if (MmDebug & MM_DBG_FORK) {
|
|
DbgPrint("removing clone descriptor at address %lx\n",CloneDescriptor);
|
|
}
|
|
#endif //DBG
|
|
|
|
|
|
// Return the pool for the global structures referenced by the
|
|
// clone descriptor.
|
|
|
|
|
|
UNLOCK_PFN (OldIrql);
|
|
|
|
if (CurrentProcess->ForkWasSuccessful == MM_FORK_SUCCEEDED) {
|
|
|
|
PsReturnPoolQuota (CurrentProcess,
|
|
PagedPool,
|
|
CloneDescriptor->PagedPoolQuotaCharge);
|
|
|
|
PsReturnPoolQuota (CurrentProcess,
|
|
NonPagedPool,
|
|
sizeof(MMCLONE_HEADER));
|
|
}
|
|
|
|
ExFreePool (CloneDescriptor);
|
|
LOCK_PFN (OldIrql);
|
|
}
|
|
|
|
return MutexReleased;
|
|
}
|
|
|
|
VOID
|
|
MiWaitForForkToComplete (
|
|
IN PEPROCESS CurrentProcess
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine waits for the current process to complete a fork
|
|
operation.
|
|
|
|
Arguments:
|
|
|
|
CurrentProcess - Supplies the current process value.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
Environment:
|
|
|
|
Kernel mode, APCs disabled, working set mutex and PFN mutex held.
|
|
|
|
--*/
|
|
|
|
{
|
|
ULONG OldIrql;
|
|
LOGICAL WsHeldSafe;
|
|
|
|
|
|
// A fork operation is in progress and the count of clone-blocks
|
|
// and other structures may not be changed. Release the mutexes
|
|
// and wait for the address creation mutex which governs the
|
|
// fork operation.
|
|
|
|
|
|
UNLOCK_PFN (APC_LEVEL);
|
|
|
|
|
|
// The working set mutex may have been acquired safely or unsafely
|
|
// by our caller. Handle both cases here and below, carefully making sure
|
|
// that the OldIrql left in the WS mutex on return is the same as on entry.
|
|
|
|
|
|
OldIrql = CurrentProcess->WorkingSetLock.OldIrql;
|
|
|
|
UNLOCK_WS_REGARDLESS (CurrentProcess, WsHeldSafe);
|
|
|
|
|
|
// Explicitly acquire the working set lock safely as it may be released and
|
|
// reacquired by our caller.
|
|
|
|
|
|
LOCK_ADDRESS_SPACE (CurrentProcess);
|
|
|
|
|
|
// The working set lock may have been acquired safely or unsafely
|
|
// by our caller. Reacquire it in the same manner our caller did.
|
|
|
|
|
|
LOCK_WS_REGARDLESS (CurrentProcess, WsHeldSafe);
|
|
|
|
|
|
// Release the address creation mutex, the working set mutex
|
|
// must be held to set the ForkInProgress field.
|
|
|
|
|
|
UNLOCK_ADDRESS_SPACE (CurrentProcess);
|
|
|
|
|
|
// Ensure the previous IRQL is correctly set to its entry value.
|
|
|
|
|
|
CurrentProcess->WorkingSetLock.OldIrql = OldIrql;
|
|
|
|
|
|
// Get the PFN lock again.
|
|
|
|
|
|
LOCK_PFN (OldIrql);
|
|
return;
|
|
}
|
|
#if DBG
|
|
VOID
|
|
CloneTreeWalk (
|
|
PMMCLONE_DESCRIPTOR Start
|
|
)
|
|
|
|
{
|
|
Start;
|
|
NodeTreeWalk ( (PMMADDRESS_NODE)(PsGetCurrentProcess()->CloneRoot));
|
|
return;
|
|
}
|
|
#endif //DBG
|
|
|
|
VOID
|
|
MiUpPfnReferenceCount (
|
|
IN PFN_NUMBER Page,
|
|
IN USHORT Count
|
|
)
|
|
|
|
// non paged helper routine.
|
|
|
|
{
|
|
KIRQL OldIrql;
|
|
PMMPFN Pfn1;
|
|
|
|
Pfn1 = MI_PFN_ELEMENT (Page);
|
|
LOCK_PFN (OldIrql);
|
|
Pfn1->u3.e2.ReferenceCount += Count;
|
|
UNLOCK_PFN (OldIrql);
|
|
return;
|
|
}
|
|
|
|
VOID
|
|
MiDownPfnReferenceCount (
|
|
IN PFN_NUMBER Page,
|
|
IN USHORT Count
|
|
)
|
|
|
|
// non paged helper routine.
|
|
|
|
{
|
|
USHORT i;
|
|
KIRQL OldIrql;
|
|
|
|
LOCK_PFN (OldIrql);
|
|
for (i = 0; i < Count; i += 1) {
|
|
MiDecrementReferenceCount (Page);
|
|
}
|
|
UNLOCK_PFN (OldIrql);
|
|
return;
|
|
}
|
|
|
|
VOID
|
|
MiUpControlAreaRefs (
|
|
IN PCONTROL_AREA ControlArea
|
|
)
|
|
|
|
{
|
|
KIRQL OldIrql;
|
|
|
|
LOCK_PFN (OldIrql);
|
|
|
|
ControlArea->NumberOfMappedViews += 1;
|
|
ControlArea->NumberOfUserReferences += 1;
|
|
|
|
UNLOCK_PFN (OldIrql);
|
|
return;
|
|
}
|
|
|
|
|
|
ULONG
|
|
MiDoneWithThisPageGetAnother (
|
|
IN PPFN_NUMBER PageFrameIndex,
|
|
IN PMMPTE PointerPde,
|
|
IN PEPROCESS CurrentProcess
|
|
)
|
|
|
|
{
|
|
KIRQL OldIrql;
|
|
ULONG ReleasedMutex;
|
|
|
|
LOCK_PFN (OldIrql);
|
|
|
|
if (*PageFrameIndex != (PFN_NUMBER)-1) {
|
|
|
|
|
|
// Decrement the share count of the last page which
|
|
// we operated on.
|
|
|
|
|
|
MiDecrementShareCountOnly (*PageFrameIndex);
|
|
}
|
|
|
|
ReleasedMutex =
|
|
MiEnsureAvailablePageOrWait (
|
|
CurrentProcess,
|
|
NULL);
|
|
|
|
*PageFrameIndex = MiRemoveZeroPage (
|
|
MI_PAGE_COLOR_PTE_PROCESS (PointerPde,
|
|
&CurrentProcess->NextPageColor));
|
|
|
|
UNLOCK_PFN (OldIrql);
|
|
return ReleasedMutex;
|
|
}
|
|
|
|
VOID
|
|
MiUpCloneProcessRefCount (
|
|
IN PMMCLONE_DESCRIPTOR Clone
|
|
)
|
|
{
|
|
KIRQL OldIrql;
|
|
|
|
LOCK_PFN (OldIrql);
|
|
|
|
Clone->CloneHeader->NumberOfProcessReferences += 1;
|
|
|
|
UNLOCK_PFN (OldIrql);
|
|
return;
|
|
}
|
|
|
|
VOID
|
|
MiUpCloneProtoRefCount (
|
|
IN PMMCLONE_BLOCK CloneProto,
|
|
IN PEPROCESS CurrentProcess
|
|
)
|
|
|
|
{
|
|
KIRQL OldIrql;
|
|
|
|
LOCK_PFN (OldIrql);
|
|
|
|
MiMakeSystemAddressValidPfnWs (CloneProto,
|
|
CurrentProcess);
|
|
|
|
CloneProto->CloneRefCount += 1;
|
|
|
|
UNLOCK_PFN (OldIrql);
|
|
return;
|
|
}
|
|
|
|
ULONG
|
|
MiHandleForkTransitionPte (
|
|
IN PMMPTE PointerPte,
|
|
IN PMMPTE PointerNewPte,
|
|
IN PMMCLONE_BLOCK ForkProtoPte
|
|
)
|
|
|
|
{
|
|
KIRQL OldIrql;
|
|
PMMPFN Pfn2;
|
|
MMPTE PteContents;
|
|
PMMPTE ContainingPte;
|
|
PFN_NUMBER PageTablePage;
|
|
MMPTE TempPte;
|
|
PMMPFN PfnForkPtePage;
|
|
|
|
|
|
LOCK_PFN (OldIrql);
|
|
|
|
|
|
// Now that we have the PFN mutex which prevents pages from
|
|
// leaving the transition state, examine the PTE again to
|
|
// ensure that it is still transition.
|
|
|
|
|
|
PteContents = *(volatile PMMPTE)PointerPte;
|
|
|
|
if ((PteContents.u.Soft.Transition == 0) ||
|
|
(PteContents.u.Soft.Prototype == 1)) {
|
|
|
|
|
|
// The PTE is no longer in transition... do this
|
|
// loop again.
|
|
|
|
|
|
UNLOCK_PFN (OldIrql);
|
|
return TRUE;
|
|
|
|
} else {
|
|
|
|
|
|
// The PTE is still in transition, handle like a
|
|
// valid PTE.
|
|
|
|
|
|
Pfn2 = MI_PFN_ELEMENT (PteContents.u.Trans.PageFrameNumber);
|
|
|
|
|
|
// Assertion that PTE is not in prototype PTE format.
|
|
|
|
|
|
ASSERT (Pfn2->u3.e1.PrototypePte != 1);
|
|
|
|
|
|
// This is a private page in transition state,
|
|
// create a fork prototype PTE
|
|
// which becomes the "prototype" PTE for this page.
|
|
|
|
|
|
ForkProtoPte->ProtoPte = PteContents;
|
|
|
|
|
|
// Make the protection write-copy if writable.
|
|
|
|
|
|
MI_MAKE_PROTECT_WRITE_COPY (ForkProtoPte->ProtoPte);
|
|
|
|
ForkProtoPte->CloneRefCount = 2;
|
|
|
|
|
|
// Transform the PFN element to reference this new fork
|
|
// prototype PTE.
|
|
|
|
|
|
|
|
// Decrement the share count for the page table
|
|
// page which contains the PTE as it is no longer
|
|
// valid or in transition.
|
|
|
|
Pfn2->PteAddress = &ForkProtoPte->ProtoPte;
|
|
Pfn2->u3.e1.PrototypePte = 1;
|
|
|
|
|
|
// Make original PTE copy on write.
|
|
|
|
|
|
MI_MAKE_PROTECT_WRITE_COPY (Pfn2->OriginalPte);
|
|
|
|
ContainingPte = MiGetPteAddress(&ForkProtoPte->ProtoPte);
|
|
|
|
if (ContainingPte->u.Hard.Valid == 0) {
|
|
#if !defined (_WIN64)
|
|
if (!NT_SUCCESS(MiCheckPdeForPagedPool (&ForkProtoPte->ProtoPte))) {
|
|
#endif
|
|
KeBugCheckEx (MEMORY_MANAGEMENT,
|
|
0x61940,
|
|
(ULONG_PTR)&ForkProtoPte->ProtoPte,
|
|
(ULONG_PTR)ContainingPte->u.Long,
|
|
(ULONG_PTR)MiGetVirtualAddressMappedByPte(&ForkProtoPte->ProtoPte));
|
|
#if !defined (_WIN64)
|
|
}
|
|
#endif
|
|
}
|
|
|
|
PageTablePage = Pfn2->PteFrame;
|
|
|
|
Pfn2->PteFrame = MI_GET_PAGE_FRAME_FROM_PTE (ContainingPte);
|
|
|
|
|
|
// Increment the share count for the page containing
|
|
// the fork prototype PTEs as we have just placed
|
|
// a transition PTE into the page.
|
|
|
|
|
|
PfnForkPtePage = MI_PFN_ELEMENT (
|
|
ContainingPte->u.Hard.PageFrameNumber );
|
|
|
|
PfnForkPtePage->u2.ShareCount += 1;
|
|
|
|
TempPte.u.Long =
|
|
MiProtoAddressForPte (Pfn2->PteAddress);
|
|
TempPte.u.Proto.Prototype = 1;
|
|
MI_WRITE_INVALID_PTE (PointerPte, TempPte);
|
|
MI_WRITE_INVALID_PTE (PointerNewPte, TempPte);
|
|
|
|
|
|
// Decrement the share count for the page table
|
|
// page which contains the PTE as it is no longer
|
|
// valid or in transition.
|
|
|
|
|
|
MiDecrementShareCount (PageTablePage);
|
|
}
|
|
UNLOCK_PFN (OldIrql);
|
|
return FALSE;
|
|
}
|
|
|
|
VOID
|
|
MiDownShareCountFlushEntireTb (
|
|
IN PFN_NUMBER PageFrameIndex
|
|
)
|
|
|
|
{
|
|
KIRQL OldIrql;
|
|
|
|
LOCK_PFN (OldIrql);
|
|
|
|
if (PageFrameIndex != (PFN_NUMBER)-1) {
|
|
|
|
|
|
// Decrement the share count of the last page which
|
|
// we operated on.
|
|
|
|
|
|
MiDecrementShareCountOnly (PageFrameIndex);
|
|
}
|
|
|
|
KeFlushEntireTb (FALSE, FALSE);
|
|
UNLOCK_PFN (OldIrql);
|
|
return;
|
|
}
|
|
|
|
VOID
|
|
MiUpForkPageShareCount(
|
|
IN PMMPFN PfnForkPtePage
|
|
)
|
|
{
|
|
KIRQL OldIrql;
|
|
|
|
LOCK_PFN (OldIrql);
|
|
PfnForkPtePage->u2.ShareCount += 1;
|
|
|
|
UNLOCK_PFN (OldIrql);
|
|
return;
|
|
}
|
|
|
|
VOID
|
|
MiBuildForkPageTable(
|
|
IN PFN_NUMBER PageFrameIndex,
|
|
IN PMMPTE PointerPde,
|
|
IN PMMPTE PointerNewPde,
|
|
IN PFN_NUMBER PdePhysicalPage,
|
|
IN PMMPFN PfnPdPage
|
|
)
|
|
{
|
|
KIRQL OldIrql;
|
|
PMMPFN Pfn1;
|
|
|
|
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
|
|
|
|
|
|
// The PFN lock must be held while initializing the
|
|
// frame to prevent those scanning the database for
|
|
// free frames from taking it after we fill in the
|
|
// u2 field.
|
|
|
|
|
|
LOCK_PFN (OldIrql);
|
|
|
|
Pfn1->OriginalPte = DemandZeroPde;
|
|
Pfn1->u2.ShareCount = 1;
|
|
Pfn1->u3.e2.ReferenceCount = 1;
|
|
Pfn1->PteAddress = PointerPde;
|
|
Pfn1->u3.e1.Modified = 1;
|
|
Pfn1->u3.e1.PageLocation = ActiveAndValid;
|
|
Pfn1->PteFrame = PdePhysicalPage;
|
|
|
|
|
|
// Increment the share count for the page containing
|
|
// this PTE as the PTE is in transition.
|
|
|
|
|
|
PfnPdPage->u2.ShareCount += 1;
|
|
|
|
|
|
// Put the PDE into the transition state as it is not
|
|
// really mapped and decrement share count does not
|
|
// put private pages into transition, only prototypes.
|
|
|
|
|
|
MI_WRITE_INVALID_PTE (PointerNewPde, TransitionPde);
|
|
|
|
|
|
// Make the PTE owned by user mode.
|
|
|
|
|
|
#ifndef _ALPHA_
|
|
MI_SET_OWNER_IN_PTE (PointerNewPde, UserMode);
|
|
#endif //_ALPHA_
|
|
PointerNewPde->u.Trans.PageFrameNumber = PageFrameIndex;
|
|
|
|
UNLOCK_PFN (OldIrql);
|
|
}
|
|
|
|
#if defined (_X86PAE_)
|
|
VOID
|
|
MiRetrievePageDirectoryFrames(
|
|
IN PFN_NUMBER RootPhysicalPage,
|
|
OUT PPFN_NUMBER PageDirectoryFrames
|
|
)
|
|
{
|
|
ULONG i;
|
|
KIRQL OldIrql;
|
|
PMMPTE PointerPte;
|
|
|
|
PointerPte = (PMMPTE)MiMapPageInHyperSpace (RootPhysicalPage, &OldIrql);
|
|
|
|
for (i = 0; i < PD_PER_SYSTEM; i += 1) {
|
|
PageDirectoryFrames[i] = MI_GET_PAGE_FRAME_FROM_PTE (PointerPte);
|
|
PointerPte += 1;
|
|
}
|
|
|
|
MiUnmapPageInHyperSpace (OldIrql);
|
|
|
|
return;
|
|
}
|
|
#endif
|