3075 lines
88 KiB
C
3075 lines
88 KiB
C
/*++
|
||
|
||
Copyright (c) 1989-1994 Microsoft Corporation
|
||
|
||
Module Name:
|
||
|
||
pool.c
|
||
|
||
Abstract:
|
||
|
||
Thie module implements the NT executive pool allocator.
|
||
|
||
Author:
|
||
|
||
Mark Lucovsky 16-feb-1989
|
||
Lou Perazzoli 31-Aug-1991 (change from binary buddy)
|
||
David N. Cutler (davec) 27-May-1994
|
||
|
||
Environment:
|
||
|
||
kernel mode only
|
||
|
||
Revision History:
|
||
|
||
--*/
|
||
|
||
#include "exp.h"
|
||
#pragma hdrstop
|
||
|
||
#undef ExAllocatePoolWithTag
|
||
#undef ExAllocatePool
|
||
#undef ExAllocatePoolWithQuota
|
||
#undef ExAllocatePoolWithQuotaTag
|
||
#undef ExFreePoolWithTag
|
||
|
||
#if DBG
|
||
BOOLEAN ExpEchoPoolCalls;
|
||
#endif //DBG
|
||
|
||
//
|
||
// Define forward referenced funtion prototypes.
|
||
//
|
||
|
||
VOID
|
||
ExpInitializePoolDescriptor(
|
||
IN PPOOL_DESCRIPTOR PoolDescriptor,
|
||
IN POOL_TYPE PoolType,
|
||
IN ULONG PoolIndex,
|
||
IN ULONG Threshold,
|
||
IN PVOID PoolLock
|
||
);
|
||
|
||
NTSTATUS
|
||
ExpSnapShotPoolPages(
|
||
IN PVOID Address,
|
||
IN ULONG Size,
|
||
IN OUT PSYSTEM_POOL_INFORMATION PoolInformation,
|
||
IN OUT PSYSTEM_POOL_ENTRY *PoolEntryInfo,
|
||
IN ULONG Length,
|
||
IN OUT PULONG RequiredLength
|
||
);
|
||
|
||
#ifdef ALLOC_PRAGMA
|
||
#pragma alloc_text(INIT, InitializePool)
|
||
#pragma alloc_text(INIT, ExpInitializePoolDescriptor)
|
||
#if DBG
|
||
#pragma alloc_text(PAGELK, ExSnapShotPool)
|
||
#pragma alloc_text(PAGELK, ExpSnapShotPoolPages)
|
||
#endif // DBG
|
||
#endif
|
||
|
||
|
||
ULONG FirstPrint;
|
||
PPOOL_TRACKER_TABLE PoolTrackTable;
|
||
PPOOL_TRACKER_BIG_PAGES PoolBigPageTable;
|
||
|
||
ULONG PoolHitTag = 0xffffff0f;
|
||
|
||
USHORT
|
||
ExpInsertPoolTracker (
|
||
ULONG Key,
|
||
ULONG Size,
|
||
POOL_TYPE PoolType
|
||
);
|
||
|
||
VOID
|
||
ExpRemovePoolTracker (
|
||
ULONG Key,
|
||
ULONG Size,
|
||
POOL_TYPE PoolType
|
||
);
|
||
|
||
PPOOL_TRACKER_BIG_PAGES
|
||
ExpAddTagForBigPages (
|
||
IN PVOID Va,
|
||
IN ULONG Key
|
||
);
|
||
|
||
ULONG
|
||
ExpFindAndRemoveTagBigPages (
|
||
IN PVOID Va
|
||
);
|
||
|
||
PVOID
|
||
ExpAllocateStringRoutine(
|
||
IN ULONG NumberOfBytes
|
||
)
|
||
{
|
||
return ExAllocatePoolWithTag(PagedPool,NumberOfBytes,'grtS');
|
||
}
|
||
|
||
BOOLEAN
|
||
ExOkayToLockRoutine(
|
||
IN PVOID Lock
|
||
);
|
||
|
||
BOOLEAN
|
||
ExOkayToLockRoutine(
|
||
IN PVOID Lock
|
||
)
|
||
{
|
||
if (KeIsExecutingDpc()) {
|
||
return FALSE;
|
||
} else {
|
||
return TRUE;
|
||
}
|
||
}
|
||
|
||
PRTL_ALLOCATE_STRING_ROUTINE RtlAllocateStringRoutine = ExpAllocateStringRoutine;
|
||
PRTL_FREE_STRING_ROUTINE RtlFreeStringRoutine = (PRTL_FREE_STRING_ROUTINE)ExFreePool;
|
||
|
||
//
|
||
// Define macros to pack and unpack a pool index.
|
||
//
|
||
|
||
#define PACK_POOL_INDEX(Index) ((UCHAR)(((Index) << 4) | (Index)))
|
||
#define UNPACK_POOL_INDEX(Index) ((ULONG)((Index) & 0xf))
|
||
|
||
//
|
||
// This structure exists in the pool descriptor structure. There is one of
|
||
// these for each pool block size
|
||
//
|
||
// The check list macro comes in three flavors there is one in the checked
|
||
// build that will assert if the doubly linked list is ill-formed. There
|
||
// is one that is noop for the shipping version and then there is one that
|
||
// can be enabled to check the list in the free build.
|
||
//
|
||
|
||
#if DBG
|
||
#define CHECK_LIST(LINE,LIST,ENTRY) \
|
||
ASSERT((LIST)->Flink->Blink == (LIST)); \
|
||
ASSERT((LIST)->Blink->Flink == (LIST));
|
||
#elif 1
|
||
#define CHECK_LIST(LINE,LIST,ENTRY) {NOTHING;}
|
||
#else
|
||
#define CHECK_LIST(LINE,LIST,ENTRY) \
|
||
if (((LIST)->Flink->Blink != (LIST)) || \
|
||
((LIST)->Blink->Flink != (LIST))) { \
|
||
KeBugCheckEx (BAD_POOL_HEADER,3,(ULONG)LIST,LINE,(ULONG)ENTRY);\
|
||
}
|
||
#endif //DBG
|
||
|
||
#define CHECK_POOL_PAGE(PAGE) \
|
||
{ \
|
||
PPOOL_HEADER P = (PPOOL_HEADER)(((ULONG)(PAGE)) & ~(PAGE_SIZE-1)); \
|
||
ULONG SIZE, LSIZE; \
|
||
BOOLEAN FOUND=FALSE; \
|
||
LSIZE = 0; \
|
||
SIZE = 0; \
|
||
do { \
|
||
if (P == (PPOOL_HEADER)PAGE) { \
|
||
FOUND = TRUE; \
|
||
} \
|
||
if (P->PreviousSize != LSIZE) { \
|
||
DbgPrint("POOL: Inconsistent size: ( %lx ) - %lx->%u != %u\n",\
|
||
PAGE, P, P->PreviousSize, LSIZE); \
|
||
DbgBreakPoint(); \
|
||
} \
|
||
LSIZE = P->BlockSize; \
|
||
SIZE += LSIZE; \
|
||
P = (PPOOL_HEADER)((PPOOL_BLOCK)P + LSIZE); \
|
||
} while ((SIZE < (PAGE_SIZE / POOL_SMALLEST_BLOCK)) && \
|
||
(PAGE_END(P) == FALSE)); \
|
||
if ((PAGE_END(P) == FALSE) || (FOUND == FALSE)) { \
|
||
DbgPrint("POOL: Inconsistent page: %lx\n",P); \
|
||
DbgBreakPoint(); \
|
||
} \
|
||
}
|
||
|
||
|
||
//
|
||
// Define the number of paged pools. This value may be overridden at boot
|
||
// time.
|
||
//
|
||
|
||
ULONG ExpNumberOfPagedPools = NUMBER_OF_PAGED_POOLS;
|
||
|
||
//
|
||
// Pool descriptors for nonpaged pool and nonpaged pool must succeed are
|
||
// static. The pool descriptors for paged pool are dynamically allocated
|
||
// since there can be more than one paged pool. There is always one more
|
||
// paged pool descriptor than there are paged pools. This descriptor is
|
||
// used when a page allocation is done for a paged pool and is the first
|
||
// descriptor in the paged ppol descriptor array.
|
||
//
|
||
|
||
POOL_DESCRIPTOR NonPagedPoolDescriptor;
|
||
POOL_DESCRIPTOR NonPagedPoolDescriptorMS;
|
||
|
||
//
|
||
// The pool vector contains an array of pointers to pool descriptors. For
|
||
// nonpaged pool and nonpaged pool must success, this is a pointer to a
|
||
// single descriptor. For page pool, this is a pointer to an array of pool
|
||
// descriptors. The pointer to the paged pool descriptor is duplicated so
|
||
// if can be found easily by the kernel debugger.
|
||
//
|
||
|
||
PPOOL_DESCRIPTOR PoolVector[NUMBER_OF_POOLS];
|
||
PPOOL_DESCRIPTOR ExpPagedPoolDescriptor;
|
||
|
||
extern KSPIN_LOCK NonPagedPoolLock;
|
||
extern KSPIN_LOCK PoolTraceLock;
|
||
|
||
volatile ULONG ExpPoolIndex = 1;
|
||
KSPIN_LOCK ExpTaggedPoolLock;
|
||
|
||
#if DBG
|
||
PSZ PoolTypeNames[MaxPoolType] = {
|
||
"NonPaged",
|
||
"Paged",
|
||
"NonPagedMustSucceed",
|
||
"NotUsed",
|
||
"NonPagedCacheAligned",
|
||
"PagedCacheAligned",
|
||
"NonPagedCacheAlignedMustS"
|
||
};
|
||
#endif //DBG
|
||
|
||
//
|
||
// Define paged and nonpaged pool lookaside descriptors.
|
||
//
|
||
|
||
SMALL_POOL_LOOKASIDE ExpSmallNPagedPoolLookasideLists[POOL_SMALL_LISTS];
|
||
|
||
#if !defined(_PPC_)
|
||
|
||
SMALL_POOL_LOOKASIDE ExpSmallPagedPoolLookasideLists[POOL_SMALL_LISTS];
|
||
|
||
#endif
|
||
|
||
//
|
||
// Two routines to check for pool that has been altered after it was freed.
|
||
//
|
||
|
||
#if DEADBEEF
|
||
|
||
_inline
|
||
VOID
|
||
ExFillFreedPool (
|
||
IN PVOID Buffer,
|
||
IN ULONG Size,
|
||
IN ULONG Tag
|
||
)
|
||
{
|
||
RtlFillMemoryUlong( Buffer, Size, Tag );
|
||
|
||
return;
|
||
}
|
||
|
||
VOID
|
||
ExCheckFreedPool (
|
||
IN ULONG LineNumber,
|
||
IN PPOOL_HEADER PoolHeader
|
||
)
|
||
{
|
||
ULONG MatchBytes;
|
||
ULONG i;
|
||
|
||
MatchBytes = RtlCompareMemoryUlong( (PCHAR)PoolHeader + 0x10,
|
||
0x20 - 0x10,
|
||
PoolHeader->PoolTag );
|
||
if (MatchBytes != 0x20 - 0x10) {
|
||
DbgPrint("EX(%d): Freed pool block %lx modified at %lx after it was freed\n",
|
||
LineNumber,
|
||
((PCHAR)PoolHeader),
|
||
((PCHAR)PoolHeader) + 0x10 + MatchBytes);
|
||
|
||
DbgBreakPoint();
|
||
}
|
||
|
||
for (i = 1; i < PoolHeader->BlockSize; i += 1) {
|
||
|
||
MatchBytes = RtlCompareMemoryUlong( (((PCHAR)PoolHeader) + (0x20 * i)),
|
||
0x20,
|
||
*((PULONG)(((PCHAR)PoolHeader) + (0x20 * i))) );
|
||
|
||
if (MatchBytes != 0x20) {
|
||
DbgPrint("EX(%d): Freed pool block %lx modified at %lx after it was freed\n",
|
||
LineNumber,
|
||
((PCHAR)PoolHeader),
|
||
((PCHAR)PoolHeader) + (0x20 * i) + MatchBytes);
|
||
|
||
DbgBreakPoint();
|
||
}
|
||
}
|
||
|
||
return;
|
||
}
|
||
|
||
#endif
|
||
|
||
|
||
//
|
||
// LOCK_POOL and LOCK_IF_PAGED_POOL are only used within this module.
|
||
//
|
||
|
||
#define LOCK_POOL(PoolDesc, LockHandle) { \
|
||
if ((PoolDesc->PoolType & BASE_POOL_TYPE_MASK) == NonPagedPool) { \
|
||
ExAcquireSpinLock(&NonPagedPoolLock, &LockHandle); \
|
||
} else { \
|
||
ExAcquireFastMutex((PFAST_MUTEX)PoolDesc->LockAddress); \
|
||
} \
|
||
}
|
||
|
||
#define LOCK_IF_PAGED_POOL(CheckType) \
|
||
if (CheckType == PagedPool) { \
|
||
ExAcquireFastMutex((PFAST_MUTEX)PoolVector[PagedPool]->LockAddress); \
|
||
}
|
||
|
||
KIRQL
|
||
ExLockPool(
|
||
IN POOL_TYPE PoolType
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function locks the pool specified by pool type.
|
||
|
||
Arguments:
|
||
|
||
PoolType - Specifies the pool that should be locked.
|
||
|
||
Return Value:
|
||
|
||
The previous IRQL is returned as the function value.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
KIRQL OldIrql;
|
||
|
||
//
|
||
// If the pool type is nonpaged, then use a spinlock to lock the
|
||
// pool. Otherwise, use a fast mutex to lock the pool.
|
||
//
|
||
|
||
if ((PoolType & BASE_POOL_TYPE_MASK) == NonPagedPool) {
|
||
ExAcquireSpinLock(NonPagedPoolDescriptor.LockAddress, &OldIrql);
|
||
|
||
} else {
|
||
ExAcquireFastMutex((PFAST_MUTEX)PoolVector[PagedPool]->LockAddress);
|
||
OldIrql = (KIRQL)((PFAST_MUTEX)(PoolVector[PagedPool]->LockAddress))->OldIrql;
|
||
}
|
||
|
||
return OldIrql;
|
||
}
|
||
|
||
|
||
//
|
||
// UNLOCK_POOL and UNLOCK_IF_PAGED_POOL are only used within this module.
|
||
//
|
||
|
||
#define UNLOCK_POOL(PoolDesc, LockHandle) { \
|
||
if ((PoolDesc->PoolType & BASE_POOL_TYPE_MASK) == NonPagedPool) { \
|
||
ExReleaseSpinLock(&NonPagedPoolLock, (KIRQL)LockHandle); \
|
||
} else { \
|
||
ExReleaseFastMutex((PFAST_MUTEX)PoolDesc->LockAddress); \
|
||
} \
|
||
}
|
||
|
||
#define UNLOCK_IF_PAGED_POOL(CheckType) \
|
||
if (CheckType == PagedPool) { \
|
||
ExReleaseFastMutex((PFAST_MUTEX)PoolVector[PagedPool]->LockAddress); \
|
||
}
|
||
|
||
VOID
|
||
ExUnlockPool(
|
||
IN POOL_TYPE PoolType,
|
||
IN KIRQL LockHandle
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function unlocks the pool specified by pool type.
|
||
|
||
|
||
Arguments:
|
||
|
||
PoolType - Specifies the pool that should be unlocked.
|
||
|
||
LockHandle - Specifies the lock handle from a previous call to
|
||
ExLockPool.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
//
|
||
// If the pool type is nonpaged, then use a spinlock to unlock the
|
||
// pool. Otherwise, use a fast mutex to unlock the pool.
|
||
//
|
||
|
||
if ((PoolType & BASE_POOL_TYPE_MASK) == NonPagedPool) {
|
||
ExReleaseSpinLock(&NonPagedPoolLock, LockHandle);
|
||
|
||
} else {
|
||
ExReleaseFastMutex((PFAST_MUTEX)PoolVector[PagedPool]->LockAddress);
|
||
}
|
||
|
||
return;
|
||
}
|
||
|
||
VOID
|
||
ExpInitializePoolDescriptor(
|
||
IN PPOOL_DESCRIPTOR PoolDescriptor,
|
||
IN POOL_TYPE PoolType,
|
||
IN ULONG PoolIndex,
|
||
IN ULONG Threshold,
|
||
IN PVOID PoolLock
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function initializes a pool descriptor.
|
||
|
||
Arguments:
|
||
|
||
PoolDescriptor - Supplies a pointer to the pool descriptor.
|
||
|
||
PoolType - Supplies the type of the pool.
|
||
|
||
PoolIndex - Supplies the pool descriptor index.
|
||
|
||
Threshold - Supplies the threshold value for the specified pool.
|
||
|
||
PoolLock - Supplies a point to the lock for the specified pool.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
ULONG Index;
|
||
|
||
//
|
||
// Initialize statistics fields, the pool type, the threshold value,
|
||
// and the lock address
|
||
//
|
||
|
||
PoolDescriptor->PoolType = PoolType;
|
||
PoolDescriptor->PoolIndex = PoolIndex;
|
||
PoolDescriptor->RunningAllocs = 0;
|
||
PoolDescriptor->RunningDeAllocs = 0;
|
||
PoolDescriptor->TotalPages = 0;
|
||
PoolDescriptor->TotalBigPages = 0;
|
||
PoolDescriptor->Threshold = Threshold;
|
||
PoolDescriptor->LockAddress = PoolLock;
|
||
|
||
//
|
||
// Initialize the allocation listheads.
|
||
//
|
||
|
||
for (Index = 0; Index < POOL_LIST_HEADS; Index += 1) {
|
||
InitializeListHead(&PoolDescriptor->ListHeads[Index]);
|
||
}
|
||
|
||
return;
|
||
}
|
||
|
||
VOID
|
||
InitializePool(
|
||
IN POOL_TYPE PoolType,
|
||
IN ULONG Threshold
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This procedure initializes a pool descriptor for the specified pool
|
||
type. Once initialized, the pool may be used for allocation and
|
||
deallocation.
|
||
|
||
This function should be called once for each base pool type during
|
||
system initialization.
|
||
|
||
Each pool descriptor contains an array of list heads for free
|
||
blocks. Each list head holds blocks which are a multiple of
|
||
the POOL_BLOCK_SIZE. The first element on the list [0] links
|
||
together free entries of size POOL_BLOCK_SIZE, the second element
|
||
[1] links together entries of POOL_BLOCK_SIZE * 2, the third
|
||
POOL_BLOCK_SIZE * 3, etc, up to the number of blocks which fit
|
||
into a page.
|
||
|
||
Arguments:
|
||
|
||
PoolType - Supplies the type of pool being initialized (e.g.
|
||
nonpaged pool, paged pool...).
|
||
|
||
Threshold - Supplies the threshold value for the specified pool.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
PPOOL_DESCRIPTOR Descriptor;
|
||
ULONG Index;
|
||
PFAST_MUTEX FastMutex;
|
||
ULONG Size;
|
||
|
||
ASSERT((PoolType & MUST_SUCCEED_POOL_TYPE_MASK) == 0);
|
||
|
||
|
||
if (PoolType == NonPagedPool) {
|
||
|
||
//
|
||
// Initialize nonpaged pools.
|
||
//
|
||
|
||
#if !DBG
|
||
if (NtGlobalFlag & FLG_POOL_ENABLE_TAGGING) {
|
||
#endif //!DBG
|
||
PoolTrackTable = MiAllocatePoolPages(NonPagedPool,
|
||
MAX_TRACKER_TABLE *
|
||
sizeof(POOL_TRACKER_TABLE));
|
||
|
||
RtlZeroMemory(PoolTrackTable, MAX_TRACKER_TABLE * sizeof(POOL_TRACKER_TABLE));
|
||
PoolBigPageTable = MiAllocatePoolPages(NonPagedPool,
|
||
MAX_BIGPAGE_TABLE *
|
||
sizeof(POOL_TRACKER_BIG_PAGES));
|
||
|
||
RtlZeroMemory(PoolBigPageTable, MAX_BIGPAGE_TABLE * sizeof(POOL_TRACKER_BIG_PAGES));
|
||
#if !DBG
|
||
}
|
||
#endif //!DBG
|
||
|
||
//
|
||
// Initialize the spinlocks for nonpaged pool.
|
||
//
|
||
|
||
KeInitializeSpinLock (&ExpTaggedPoolLock);
|
||
KeInitializeSpinLock(&NonPagedPoolLock);
|
||
KeInitializeSpinLock(&PoolTraceLock);
|
||
|
||
//
|
||
// Initialize the nonpaged pool descriptor.
|
||
//
|
||
|
||
PoolVector[NonPagedPool] = &NonPagedPoolDescriptor;
|
||
ExpInitializePoolDescriptor(&NonPagedPoolDescriptor,
|
||
NonPagedPool,
|
||
0,
|
||
Threshold,
|
||
(PVOID)&NonPagedPoolLock);
|
||
|
||
//
|
||
// Initialize the nonpaged must succeed pool descriptor.
|
||
//
|
||
|
||
PoolVector[NonPagedPoolMustSucceed] = &NonPagedPoolDescriptorMS;
|
||
ExpInitializePoolDescriptor(&NonPagedPoolDescriptorMS,
|
||
NonPagedPoolMustSucceed,
|
||
0,
|
||
0,
|
||
(PVOID)&NonPagedPoolLock);
|
||
|
||
#if DBG
|
||
if (MmSpecialPoolTag != 0) {
|
||
MmInitializeSpecialPool();
|
||
}
|
||
#endif //DBG
|
||
|
||
} else {
|
||
|
||
//
|
||
// Allocate memory for the paged pool descriptors and fast mutexes.
|
||
//
|
||
|
||
Size = (ExpNumberOfPagedPools + 1) * (sizeof(FAST_MUTEX) + sizeof(POOL_DESCRIPTOR));
|
||
Descriptor = (PPOOL_DESCRIPTOR)ExAllocatePoolWithTag (NonPagedPoolMustSucceed,
|
||
Size,
|
||
'looP');
|
||
if (PoolTrackTable) {
|
||
ExpInsertPoolTracker('looP',
|
||
MAX_TRACKER_TABLE * sizeof(POOL_TRACKER_TABLE),
|
||
NonPagedPool);
|
||
|
||
ExpInsertPoolTracker('looP',
|
||
MAX_BIGPAGE_TABLE * sizeof(POOL_TRACKER_BIG_PAGES),
|
||
NonPagedPool);
|
||
}
|
||
|
||
FastMutex = (PFAST_MUTEX)(Descriptor + ExpNumberOfPagedPools + 1);
|
||
PoolVector[PagedPool] = Descriptor;
|
||
ExpPagedPoolDescriptor = Descriptor;
|
||
for (Index = 0; Index < (ExpNumberOfPagedPools + 1); Index += 1) {
|
||
ExInitializeFastMutex(FastMutex);
|
||
ExpInitializePoolDescriptor(Descriptor,
|
||
PagedPool,
|
||
Index,
|
||
Threshold,
|
||
(PVOID)FastMutex);
|
||
|
||
Descriptor += 1;
|
||
FastMutex += 1;
|
||
}
|
||
}
|
||
|
||
//
|
||
// The maximum cache alignment must be less than the size of the
|
||
// smallest pool block because the lower bits are being cleared
|
||
// in ExFreePool to find the entry's address.
|
||
//
|
||
|
||
#if POOL_CACHE_SUPPORTED
|
||
|
||
//
|
||
// Compute pool cache information.
|
||
//
|
||
|
||
PoolCacheSize = HalGetDmaAlignmentRequirement();
|
||
|
||
ASSERT(PoolCacheSize >= POOL_OVERHEAD);
|
||
|
||
PoolCacheOverhead = PoolCacheSize + PoolCacheSize - (sizeof(POOL_HEADER) + 1);
|
||
|
||
#ifndef CHECK_POOL_TAIL
|
||
|
||
PoolBuddyMax =
|
||
(POOL_PAGE_SIZE - (POOL_OVERHEAD + (3*POOL_SMALLEST_BLOCK) + 2*PoolCacheSize));
|
||
|
||
#else
|
||
|
||
PoolBuddyMax =
|
||
(POOL_PAGE_SIZE - (POOL_OVERHEAD + 2*PoolCacheSize + (4*POOL_SMALLEST_BLOCK)));
|
||
|
||
#endif // CHECK_POOL_TAIL
|
||
|
||
#endif //POOL_CACHE_SUPPORTED
|
||
|
||
return;
|
||
}
|
||
|
||
#if DBG
|
||
|
||
|
||
VOID
|
||
ExpVerifyPool(
|
||
PPOOL_DESCRIPTOR PoolDescriptor
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function verifies the specified pool
|
||
|
||
Arguments:
|
||
|
||
PoolDesc - Supplies a pointer to a pool descriptor.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
PLIST_ENTRY Entry;
|
||
ULONG Index;
|
||
PLIST_ENTRY ListHead;
|
||
ULONG Number;
|
||
PPOOL_HEADER PoolHeader;
|
||
|
||
//
|
||
// Scan each of the allocation lists and perform the following checks:
|
||
//
|
||
// 1. Make sure each free block is in the correct list.
|
||
//
|
||
// 2. Make sure each free block is really free.
|
||
//
|
||
// 3. Make sure all the blocks in each page add up to a page.
|
||
//
|
||
//
|
||
|
||
for (Index = 0; Index < POOL_LIST_HEADS; Index += 1) {
|
||
ListHead = &PoolDescriptor->ListHeads[Index];
|
||
Entry = ListHead->Flink;
|
||
while (Entry != ListHead) {
|
||
PoolHeader = (PPOOL_HEADER)((PCHAR)Entry - POOL_OVERHEAD);
|
||
|
||
//
|
||
// Assert that the pool block is not allocated.
|
||
//
|
||
|
||
ASSERT(PoolHeader->PoolType == 0);
|
||
|
||
ASSERT(PoolHeader->PoolIndex == PACK_POOL_INDEX(PoolDescriptor->PoolIndex));
|
||
|
||
//
|
||
// Assert that the pool block is in the correct list.
|
||
//
|
||
|
||
Number = PoolHeader->BlockSize;
|
||
if (Number > POOL_SMALL_LISTS) {
|
||
Number = (Number >> SHIFT_OFFSET) + POOL_SMALL_LISTS + 1;
|
||
}
|
||
|
||
ASSERT(Index == (Number - 1));
|
||
|
||
//
|
||
// Check to make sure the pool block is properly filled.
|
||
//
|
||
|
||
#if DEADBEEF
|
||
|
||
if (NtGlobalFlag & FLG_POOL_ENABLE_FREE_CHECK) {
|
||
|
||
//
|
||
// Make sure the pool block has not been modified after it was freed.
|
||
//
|
||
|
||
ExCheckFreedPool( __LINE__, PoolHeader );
|
||
|
||
CHECK_POOL_PAGE(PoolHeader);
|
||
}
|
||
|
||
#endif //DEADBEEF
|
||
|
||
Entry = Entry->Flink;
|
||
}
|
||
}
|
||
|
||
return;
|
||
}
|
||
|
||
#else
|
||
|
||
#define ExpVerifyPool(PoolDesc)
|
||
|
||
#endif
|
||
|
||
PVOID
|
||
ExAllocatePool(
|
||
IN POOL_TYPE PoolType,
|
||
IN ULONG NumberOfBytes
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function allocates a block of pool of the specified type and
|
||
returns a pointer to the allocated block. This function is used to
|
||
access both the page-aligned pools, and the list head entries (less than
|
||
a page) pools.
|
||
|
||
If the number of bytes specifies a size that is too large to be
|
||
satisfied by the appropriate list, then the page-aligned
|
||
pool allocator is used. The allocated block will be page-aligned
|
||
and a page-sized multiple.
|
||
|
||
Otherwise, the appropriate pool list entry is used. The allocated
|
||
block will be 64-bit aligned, but will not be page aligned. The
|
||
pool allocator calculates the smallest number of POOL_BLOCK_SIZE
|
||
that can be used to satisfy the request. If there are no blocks
|
||
available of this size, then a block of the next larger block size
|
||
is allocated and split. One piece is placed back into the pool, and
|
||
the other piece is used to satisfy the request. If the allocator
|
||
reaches the paged-sized block list, and nothing is there, the
|
||
page-aligned pool allocator is called. The page is split and added
|
||
to the pool...
|
||
|
||
Arguments:
|
||
|
||
PoolType - Supplies the type of pool to allocate. If the pool type
|
||
is one of the "MustSucceed" pool types, then this call will
|
||
always succeed and return a pointer to allocated pool.
|
||
Otherwise, if the system can not allocate the requested amount
|
||
of memory a NULL is returned.
|
||
|
||
Valid pool types:
|
||
|
||
NonPagedPool
|
||
PagedPool
|
||
NonPagedPoolMustSucceed,
|
||
NonPagedPoolCacheAligned
|
||
PagedPoolCacheAligned
|
||
NonPagedPoolCacheAlignedMustS
|
||
|
||
NumberOfBytes - Supplies the number of bytes to allocate.
|
||
|
||
Return Value:
|
||
|
||
NULL - The PoolType is not one of the "MustSucceed" pool types, and
|
||
not enough pool exists to satisfy the request.
|
||
|
||
NON-NULL - Returns a pointer to the allocated pool.
|
||
|
||
--*/
|
||
|
||
{
|
||
return ExAllocatePoolWithTag(PoolType, NumberOfBytes, 'enoN');
|
||
}
|
||
|
||
|
||
PVOID
|
||
ExAllocatePoolWithTag(
|
||
IN POOL_TYPE PoolType,
|
||
IN ULONG NumberOfBytes,
|
||
IN ULONG Tag
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function allocates a block of pool of the specified type and
|
||
returns a pointer to the allocated block. This function is used to
|
||
access both the page-aligned pools and the list head entries (less
|
||
than a page) pools.
|
||
|
||
If the number of bytes specifies a size that is too large to be
|
||
satisfied by the appropriate list, then the page-aligned pool
|
||
allocator is used. The allocated block will be page-aligned and a
|
||
page-sized multiple.
|
||
|
||
Otherwise, the appropriate pool list entry is used. The allocated
|
||
block will be 64-bit aligned, but will not be page aligned. The
|
||
pool allocator calculates the smallest number of POOL_BLOCK_SIZE
|
||
that can be used to satisfy the request. If there are no blocks
|
||
available of this size, then a block of the next larger block size
|
||
is allocated and split. One piece is placed back into the pool, and
|
||
the other piece is used to satisfy the request. If the allocator
|
||
reaches the paged-sized block list, and nothing is there, the
|
||
page-aligned pool allocator is called. The page is split and added
|
||
to the pool.
|
||
|
||
Arguments:
|
||
|
||
PoolType - Supplies the type of pool to allocate. If the pool type
|
||
is one of the "MustSucceed" pool types, then this call will
|
||
always succeed and return a pointer to allocated pool. Otherwise,
|
||
if the system can not allocate the requested amount of memory a
|
||
NULL is returned.
|
||
|
||
Valid pool types:
|
||
|
||
NonPagedPool
|
||
PagedPool
|
||
NonPagedPoolMustSucceed,
|
||
NonPagedPoolCacheAligned
|
||
PagedPoolCacheAligned
|
||
NonPagedPoolCacheAlignedMustS
|
||
|
||
NumberOfBytes - Supplies the number of bytes to allocate.
|
||
|
||
Return Value:
|
||
|
||
NULL - The PoolType is not one of the "MustSucceed" pool types, and
|
||
not enough pool exists to satisfy the request.
|
||
|
||
NON-NULL - Returns a pointer to the allocated pool.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
PVOID Block;
|
||
PPOOL_HEADER Entry;
|
||
PSMALL_POOL_LOOKASIDE LookasideList;
|
||
PPOOL_HEADER NextEntry;
|
||
PPOOL_HEADER SplitEntry;
|
||
KIRQL LockHandle;
|
||
PPOOL_DESCRIPTOR PoolDesc;
|
||
PVOID Lock;
|
||
ULONG Index;
|
||
ULONG ListNumber;
|
||
ULONG NeededSize;
|
||
ULONG PoolIndex;
|
||
POOL_TYPE CheckType;
|
||
PLIST_ENTRY ListHead;
|
||
USHORT PoolTagHash;
|
||
PKPRCB Prcb;
|
||
POOL_TYPE NewPoolType;
|
||
|
||
#if POOL_CACHE_SUPPORTED
|
||
ULONG CacheOverhead;
|
||
#else
|
||
#define CacheOverhead POOL_OVERHEAD
|
||
#endif
|
||
|
||
#if DBG
|
||
VOID
|
||
CalculatePoolUtilization(
|
||
IN PPOOL_DESCRIPTOR PoolDesc,
|
||
IN ULONG BytesWanted
|
||
);
|
||
#endif
|
||
|
||
|
||
ASSERT(NumberOfBytes != 0);
|
||
|
||
//
|
||
// Isolate the base pool type and select a pool from which to allocate
|
||
// the specified block size.
|
||
//
|
||
|
||
CheckType = PoolType & BASE_POOL_TYPE_MASK;
|
||
PoolDesc = PoolVector[CheckType];
|
||
|
||
//
|
||
// Check to determine if the requested block can be allocated from one
|
||
// of the pool lists or must be directed allocated from virtual memory.
|
||
//
|
||
|
||
if (NumberOfBytes > POOL_BUDDY_MAX) {
|
||
|
||
//
|
||
// The requested size is greater than the largest block maintained
|
||
// by allocation lists.
|
||
//
|
||
|
||
ASSERT((PoolType & MUST_SUCCEED_POOL_TYPE_MASK) == 0);
|
||
|
||
LOCK_POOL(PoolDesc, LockHandle);
|
||
|
||
PoolDesc->RunningAllocs++;
|
||
Entry = (PPOOL_HEADER)MiAllocatePoolPages(CheckType, NumberOfBytes);
|
||
UNLOCK_POOL(PoolDesc, LockHandle);
|
||
if (Entry != NULL) {
|
||
PPOOL_TRACKER_BIG_PAGES p;
|
||
|
||
PoolDesc->TotalBigPages += BYTES_TO_PAGES(NumberOfBytes);
|
||
if (PoolBigPageTable != FALSE) {
|
||
if (!(p = ExpAddTagForBigPages((PVOID)Entry, Tag))) {
|
||
Tag = ' GIB';
|
||
}
|
||
#if DBG || (i386 && !FPO)
|
||
else
|
||
if ((NtGlobalFlag & FLG_KERNEL_STACK_TRACE_DB) && Entry) {
|
||
p->NumberOfPages = (USHORT)BYTES_TO_PAGES(NumberOfBytes);
|
||
#if i386 && !FPO
|
||
p->AllocatorBackTraceIndex = RtlLogStackBackTrace();
|
||
#endif // i386 && !FPO
|
||
}
|
||
#endif // DBG || (i386 && !FPO)
|
||
|
||
ExpInsertPoolTracker(Tag, ROUND_TO_PAGES(NumberOfBytes), PoolType);
|
||
}
|
||
|
||
} else {
|
||
KdPrint(("EX: ExAllocatePool( %d ) returning NULL\n",NumberOfBytes));
|
||
|
||
if ((PoolType & POOL_RAISE_IF_ALLOCATION_FAILURE) != 0) {
|
||
ExRaiseStatus(STATUS_INSUFFICIENT_RESOURCES);
|
||
}
|
||
}
|
||
|
||
#if DBG
|
||
|
||
if (ExpEchoPoolCalls){
|
||
PVOID CallingAddress;
|
||
PVOID CallersCaller;
|
||
|
||
DbgPrint("0x%lx EXALLOC: from %s size %d",
|
||
Entry,
|
||
PoolTypeNames[PoolType & MaxPoolType],
|
||
NumberOfBytes
|
||
);
|
||
|
||
RtlGetCallersAddress(&CallingAddress, &CallersCaller);
|
||
DbgPrint(" Callers:%lx, %lx\n", CallingAddress, CallersCaller);
|
||
}
|
||
|
||
#endif //DBG
|
||
|
||
return Entry;
|
||
|
||
} else {
|
||
|
||
//
|
||
// The requested size is less than of equal to the size of the
|
||
// maximum block maintained by the allocation lists.
|
||
//
|
||
|
||
#if DBG
|
||
if ((MmSpecialPoolTag != 0) && (Tag == MmSpecialPoolTag)) {
|
||
return MmAllocateSpecialPool(NumberOfBytes, Tag);
|
||
}
|
||
#endif //DBG
|
||
|
||
//
|
||
// If the request is for cache aligned memory adjust the number of
|
||
// bytes.
|
||
//
|
||
|
||
#if POOL_CACHE_SUPPORTED //only compile for machines which have a nonzero value.
|
||
|
||
CacheOverhead = POOL_OVERHEAD;
|
||
if (PoolType & CACHE_ALIGNED_POOL_TYPE_MASK) {
|
||
NumberOfBytes += PoolCacheOverhead;
|
||
CacheOverhead = PoolCacheSize;
|
||
}
|
||
|
||
#endif //POOL_CACHE_SUPPORTED
|
||
|
||
//
|
||
// Compute the Index of the listhead for blocks of the requested
|
||
// size.
|
||
//
|
||
|
||
ListNumber = ((NumberOfBytes + POOL_OVERHEAD + (POOL_SMALLEST_BLOCK - 1)) >> POOL_BLOCK_SHIFT);
|
||
|
||
#ifdef CHECK_POOL_TAIL
|
||
|
||
ListNumber += 1;
|
||
|
||
#endif // CHECK_POOL_TAIL
|
||
|
||
|
||
NeededSize = ListNumber;
|
||
if (ListNumber > POOL_SMALL_LISTS) {
|
||
ListNumber = (ListNumber >> SHIFT_OFFSET) + POOL_SMALL_LISTS + 2;
|
||
}
|
||
|
||
//
|
||
// If the pool type is paged, then pick a starting pool number and
|
||
// attempt to lock each paged pool in circular succession. Otherwise,
|
||
// lock the nonpaged pool as the same lock is used for both nonpaged
|
||
// and nonpage must succeed.
|
||
//
|
||
// N.B. The paged pool is selected in a round robin fashion using a
|
||
// simple counter. Note that the counter is incremented using a
|
||
// a noninterlocked sequence, but the pool index is never allowed
|
||
// to get out of range.
|
||
//
|
||
|
||
Prcb = KeGetCurrentPrcb();
|
||
if (CheckType == PagedPool) {
|
||
|
||
//
|
||
// If the requested pool block is a small block, then attempt to
|
||
// allocate the requested pool from the per processor single entry
|
||
// lookaside list. If the allocation attempt fails, then select a
|
||
// pool to alocate from and allocate the block normally.
|
||
//
|
||
|
||
#if !defined(CHECK_POOL_TAIL) && (DEADBEEF == 0)
|
||
|
||
if (NeededSize <= POOL_SMALL_LISTS) {
|
||
|
||
#if defined(_PPC_)
|
||
|
||
if ((Entry = (PPOOL_HEADER)InterlockedExchange((PLONG)&Prcb->PagedFreeEntry[NeededSize - 1],
|
||
(LONG)NULL)) != NULL) {
|
||
|
||
PoolIndex = UNPACK_POOL_INDEX(Entry->PoolIndex);
|
||
Prcb->PagedPoolLookasideHits += 1;
|
||
|
||
#else
|
||
|
||
LookasideList = &ExpSmallPagedPoolLookasideLists[NeededSize - 1];
|
||
LookasideList->TotalAllocates += 1;
|
||
if ((Isx86FeaturePresent(KF_CMPXCHG8B)) &&
|
||
(Entry = (PPOOL_HEADER)ExInterlockedPopEntrySList(&LookasideList->SListHead,
|
||
&LookasideList->Lock)) != NULL) {
|
||
|
||
|
||
Entry -= 1;
|
||
LookasideList->AllocateHits += 1;
|
||
|
||
#endif
|
||
NewPoolType = (PoolType & (BASE_POOL_TYPE_MASK | POOL_QUOTA_MASK)) + 1;
|
||
#if defined (_ALPHA_)
|
||
//
|
||
// On Alpha, Entry->PoolType cannot be updated without synchronizing with
|
||
// updates to Entry->PreviousSize. Otherwise, the lack of byte granularity
|
||
// can cause one update to get lost. In order to avoid an expensive interlocked
|
||
// operation, check PoolType to see if it really needs to be updated.
|
||
//
|
||
if (Entry->PoolType != NewPoolType) {
|
||
ULONG NewHeader;
|
||
ULONG OldHeader;
|
||
|
||
NewHeader = Entry->Ulong1;
|
||
do {
|
||
OldHeader = NewHeader;
|
||
((PPOOL_HEADER)(&NewHeader))->PoolType = NewPoolType;
|
||
NewHeader = (ULONG)InterlockedCompareExchange((PVOID *)&Entry->Ulong1,
|
||
(PVOID)NewHeader,
|
||
(PVOID)OldHeader);
|
||
|
||
} while ( NewHeader != OldHeader );
|
||
}
|
||
#else
|
||
Entry->PoolType = NewPoolType;
|
||
#endif
|
||
|
||
if (PoolTrackTable != NULL) {
|
||
ExpInsertPoolTracker(Tag,
|
||
Entry->BlockSize << POOL_BLOCK_SHIFT,
|
||
PoolType);
|
||
|
||
}
|
||
|
||
Entry->PoolTag = Tag;
|
||
return (PUCHAR)Entry + CacheOverhead;
|
||
}
|
||
}
|
||
|
||
#endif
|
||
|
||
//
|
||
// If there is more than one paged pool, then attempt to find
|
||
// one that can be immediately locked.
|
||
//
|
||
|
||
PoolIndex = 1;
|
||
if (ExpNumberOfPagedPools != PoolIndex) {
|
||
ExpPoolIndex += 1;
|
||
PoolIndex = ExpPoolIndex;
|
||
if (PoolIndex > ExpNumberOfPagedPools) {
|
||
PoolIndex = 1;
|
||
ExpPoolIndex = 1;
|
||
}
|
||
|
||
Index = PoolIndex;
|
||
do {
|
||
Lock = PoolDesc[PoolIndex].LockAddress;
|
||
if (ExTryToAcquireFastMutex((PFAST_MUTEX)Lock) != FALSE) {
|
||
goto PoolLocked;
|
||
}
|
||
|
||
PoolIndex += 1;
|
||
if (PoolIndex > ExpNumberOfPagedPools) {
|
||
PoolIndex = 1;
|
||
}
|
||
|
||
} while (PoolIndex != Index);
|
||
}
|
||
|
||
//
|
||
// None of the paged pools could be conditionally locked or there
|
||
// is only one paged pool. The first pool considered is picked as
|
||
// the victim to wait on.
|
||
//
|
||
|
||
Lock = PoolDesc[PoolIndex].LockAddress;
|
||
ExAcquireFastMutex((PFAST_MUTEX)Lock);
|
||
|
||
PoolLocked:
|
||
PoolDesc = &PoolDesc[PoolIndex];
|
||
|
||
} else {
|
||
|
||
//
|
||
// If the requested pool block is a small block, then attempt to
|
||
// allocate the requested pool from the per processor single entry
|
||
// lookaside list. If the allocation attempt fails, then allocate
|
||
// the pool normally.
|
||
//
|
||
|
||
#if !defined(CHECK_POOL_TAIL) && (DEADBEEF == 0)
|
||
|
||
if (NeededSize <= POOL_SMALL_LISTS) {
|
||
LookasideList = &ExpSmallNPagedPoolLookasideLists[NeededSize - 1];
|
||
LookasideList->TotalAllocates += 1;
|
||
if ((Entry = (PPOOL_HEADER)ExInterlockedPopEntrySList(&LookasideList->SListHead,
|
||
&LookasideList->Lock)) != NULL) {
|
||
|
||
|
||
Entry -= 1;
|
||
LookasideList->AllocateHits += 1;
|
||
NewPoolType = (PoolType & (BASE_POOL_TYPE_MASK | POOL_QUOTA_MASK)) + 1;
|
||
#if defined (_ALPHA_)
|
||
//
|
||
// On Alpha, Entry->PoolType cannot be updated without synchronizing with
|
||
// updates to Entry->PreviousSize. Otherwise, the lack of byte granularity
|
||
// can cause one update to get lost. In order to avoid an expensive interlocked
|
||
// operation, check PoolType to see if it really needs to be updated.
|
||
//
|
||
if (Entry->PoolType != NewPoolType) {
|
||
ULONG NewHeader;
|
||
ULONG OldHeader;
|
||
|
||
NewHeader = Entry->Ulong1;
|
||
do {
|
||
OldHeader = NewHeader;
|
||
((PPOOL_HEADER)(&NewHeader))->PoolType = NewPoolType;
|
||
NewHeader = (ULONG)InterlockedCompareExchange((PVOID *)&Entry->Ulong1,
|
||
(PVOID)NewHeader,
|
||
(PVOID)OldHeader);
|
||
|
||
} while ( NewHeader != OldHeader );
|
||
}
|
||
#else
|
||
Entry->PoolType = NewPoolType;
|
||
#endif
|
||
if (PoolTrackTable != NULL) {
|
||
ExpInsertPoolTracker(Tag,
|
||
Entry->BlockSize << POOL_BLOCK_SHIFT,
|
||
PoolType);
|
||
|
||
}
|
||
|
||
Entry->PoolTag = Tag;
|
||
return (PUCHAR)Entry + CacheOverhead;
|
||
}
|
||
}
|
||
|
||
#endif
|
||
|
||
PoolIndex = 0;
|
||
ExAcquireSpinLock(&NonPagedPoolLock, &LockHandle);
|
||
}
|
||
|
||
ASSERT(PoolIndex == PoolDesc->PoolIndex);
|
||
|
||
//
|
||
// The following code has an outer loop and an inner loop.
|
||
//
|
||
// The outer loop is utilized to repeat a nonpaged must succeed
|
||
// allocation if necessary.
|
||
//
|
||
// The inner loop is used to repeat an allocation attempt if there
|
||
// are no entries in any of the pool lists.
|
||
//
|
||
|
||
PoolDesc->RunningAllocs += 1;
|
||
ListHead = &PoolDesc->ListHeads[ListNumber - 1];
|
||
do {
|
||
|
||
//
|
||
// Attempt to allocate the requested block from the current free
|
||
// blocks.
|
||
//
|
||
|
||
do {
|
||
|
||
//
|
||
// If the list is not empty, then allocate a block from the
|
||
// selected list.
|
||
//
|
||
|
||
if (IsListEmpty(ListHead) == FALSE) {
|
||
|
||
CHECK_LIST( __LINE__, ListHead, 0 );
|
||
Block = RemoveHeadList(ListHead);
|
||
CHECK_LIST( __LINE__, ListHead, 0 );
|
||
|
||
Entry = (PPOOL_HEADER)((PCHAR)Block - POOL_OVERHEAD);
|
||
|
||
ASSERT(Entry->BlockSize >= NeededSize);
|
||
|
||
ASSERT(Entry->PoolIndex == PACK_POOL_INDEX(PoolIndex));
|
||
|
||
ASSERT(Entry->PoolType == 0);
|
||
|
||
if (Entry->BlockSize != NeededSize) {
|
||
|
||
//
|
||
// The selected block is larger than the allocation
|
||
// request. Split the block and insert the remaining
|
||
// fragment in the appropriate list.
|
||
//
|
||
// If the entry is at the start of a page, then take
|
||
// the allocation from the front of the block so as
|
||
// to minimize fragmentation. Otherwise, take the
|
||
// allocation from the end of the block which may
|
||
// also reduce fragmentation is the block is at the
|
||
// end of a page.
|
||
//
|
||
|
||
if (Entry->PreviousSize == 0) {
|
||
|
||
//
|
||
// The entry is at the start of a page.
|
||
//
|
||
|
||
SplitEntry = (PPOOL_HEADER)((PPOOL_BLOCK)Entry + NeededSize);
|
||
SplitEntry->BlockSize = Entry->BlockSize - (UCHAR)NeededSize;
|
||
SplitEntry->PreviousSize = (UCHAR)NeededSize;
|
||
|
||
//
|
||
// If the allocated block is not at the end of a
|
||
// page, then adjust the size of the next block.
|
||
//
|
||
|
||
NextEntry = (PPOOL_HEADER)((PPOOL_BLOCK)SplitEntry + SplitEntry->BlockSize);
|
||
if (PAGE_END(NextEntry) == FALSE) {
|
||
NextEntry->PreviousSize = SplitEntry->BlockSize;
|
||
}
|
||
|
||
} else {
|
||
|
||
//
|
||
// The entry is not at the start of a page.
|
||
//
|
||
|
||
SplitEntry = Entry;
|
||
Entry->BlockSize -= (UCHAR)NeededSize;
|
||
Entry = (PPOOL_HEADER)((PPOOL_BLOCK)Entry + Entry->BlockSize);
|
||
Entry->PreviousSize = SplitEntry->BlockSize;
|
||
|
||
//
|
||
// If the allocated block is not at the end of a
|
||
// page, then adjust the size of the next block.
|
||
//
|
||
|
||
NextEntry = (PPOOL_HEADER)((PPOOL_BLOCK)Entry + NeededSize);
|
||
if (PAGE_END(NextEntry) == FALSE) {
|
||
NextEntry->PreviousSize = (UCHAR)NeededSize;
|
||
}
|
||
}
|
||
|
||
//
|
||
// Set the size of the allocated entry, clear the pool
|
||
// type of the split entry, set the index of the split
|
||
// entry, and insert the split entry in the appropriate
|
||
// free list.
|
||
//
|
||
|
||
Entry->BlockSize = (UCHAR)NeededSize;
|
||
Entry->PoolIndex = PACK_POOL_INDEX(PoolIndex);
|
||
SplitEntry->PoolType = 0;
|
||
SplitEntry->PoolIndex = PACK_POOL_INDEX(PoolIndex);
|
||
Index = SplitEntry->BlockSize;
|
||
if (Index > POOL_SMALL_LISTS) {
|
||
Index = (Index >> SHIFT_OFFSET) + POOL_SMALL_LISTS + 1;
|
||
}
|
||
|
||
InsertTailList(&PoolDesc->ListHeads[Index - 1],
|
||
((PLIST_ENTRY)((PCHAR)SplitEntry + POOL_OVERHEAD)));
|
||
}
|
||
|
||
Entry->PoolType = (PoolType & (BASE_POOL_TYPE_MASK | POOL_QUOTA_MASK)) + 1;
|
||
|
||
#if DEADBEEF
|
||
|
||
if (NtGlobalFlag & FLG_POOL_ENABLE_FREE_CHECK) {
|
||
|
||
//
|
||
// Make sure the pool block has not been modified after it was freed
|
||
// and then fill it in with allocated tags.
|
||
//
|
||
|
||
ExCheckFreedPool( __LINE__, Entry );
|
||
|
||
RtlFillMemoryUlong( (PCHAR)Entry + POOL_OVERHEAD,
|
||
(Entry->BlockSize << POOL_BLOCK_SHIFT) - POOL_OVERHEAD,
|
||
ALLOCATED_POOL );
|
||
|
||
CHECK_POOL_PAGE(Entry);
|
||
}
|
||
|
||
#endif //DEADBEEF
|
||
|
||
if (PoolTrackTable != NULL) {
|
||
PoolTagHash = ExpInsertPoolTracker(Tag,
|
||
Entry->BlockSize << POOL_BLOCK_SHIFT,
|
||
PoolType);
|
||
|
||
}
|
||
|
||
UNLOCK_POOL(PoolDesc, LockHandle);
|
||
|
||
Entry->PoolTag = Tag;
|
||
|
||
#if i386 && !FPO
|
||
if (NtGlobalFlag & FLG_KERNEL_STACK_TRACE_DB) {
|
||
USHORT AllocatorBackTraceIndex;
|
||
|
||
if (AllocatorBackTraceIndex = RtlLogStackBackTrace()) {
|
||
Entry->AllocatorBackTraceIndex = AllocatorBackTraceIndex |
|
||
POOL_BACKTRACEINDEX_PRESENT;
|
||
Entry->PoolTagHash = PoolTagHash;
|
||
}
|
||
}
|
||
#endif // i386 && !FPO
|
||
|
||
#if DBG
|
||
if (ExpEchoPoolCalls){
|
||
PVOID CallingAddress;
|
||
PVOID CallersCaller;
|
||
|
||
DbgPrint("0x%lx EXALLOC: from %s size %d",
|
||
((PCH)Entry + POOL_OVERHEAD),
|
||
PoolTypeNames[PoolType & MaxPoolType],
|
||
NumberOfBytes
|
||
);
|
||
|
||
RtlGetCallersAddress(&CallingAddress, &CallersCaller);
|
||
DbgPrint(" Callers:%lx, %lx\n", CallingAddress, CallersCaller);
|
||
}
|
||
#endif
|
||
|
||
return (PCHAR)Entry + CacheOverhead;
|
||
|
||
} else {
|
||
ListHead += 1;
|
||
}
|
||
|
||
} while (ListHead != &PoolDesc->ListHeads[POOL_LIST_HEADS]);
|
||
|
||
//
|
||
// A block of the desired size does not exist and there are
|
||
// no large blocks that can be split to satify the allocation.
|
||
// Attempt to expand the pool by allocating another page to be
|
||
// added to the pool.
|
||
//
|
||
// If the pool type is paged pool, then the paged pool page lock
|
||
// must be held during the allocation of the pool pages.
|
||
//
|
||
|
||
LOCK_IF_PAGED_POOL(CheckType);
|
||
|
||
Entry = (PPOOL_HEADER)MiAllocatePoolPages(CheckType, PAGE_SIZE);
|
||
|
||
UNLOCK_IF_PAGED_POOL(CheckType);
|
||
|
||
if (Entry == NULL) {
|
||
if ((PoolType & MUST_SUCCEED_POOL_TYPE_MASK) != 0) {
|
||
|
||
//
|
||
// Must succeed pool was requested. Reset the the type,
|
||
// the pool descriptor address, and continue the search.
|
||
//
|
||
|
||
CheckType = NonPagedPoolMustSucceed;
|
||
PoolDesc = PoolVector[NonPagedPoolMustSucceed];
|
||
ListHead = &PoolDesc->ListHeads[ListNumber - 1];
|
||
continue;
|
||
|
||
} else {
|
||
|
||
//
|
||
// No more pool of the specified type is available.
|
||
//
|
||
|
||
KdPrint(("EX: ExAllocatePool( %d ) returning NULL\n",
|
||
NumberOfBytes));
|
||
|
||
UNLOCK_POOL(PoolDesc, LockHandle);
|
||
|
||
if ((PoolType & POOL_RAISE_IF_ALLOCATION_FAILURE) != 0) {
|
||
ExRaiseStatus(STATUS_INSUFFICIENT_RESOURCES);
|
||
}
|
||
|
||
return NULL;
|
||
}
|
||
}
|
||
|
||
#if DEADBEEF
|
||
|
||
if (NtGlobalFlag & FLG_POOL_ENABLE_FREE_CHECK) {
|
||
|
||
//
|
||
// Fill in the entire page with freed pool tags.
|
||
//
|
||
|
||
ExFillFreedPool( (PCHAR)Entry + POOL_OVERHEAD,
|
||
PAGE_SIZE - POOL_OVERHEAD,
|
||
FREED_POOL );
|
||
|
||
Entry->PoolTag = FREED_POOL;
|
||
}
|
||
|
||
#endif
|
||
|
||
//
|
||
// Insert the allocate page in the last allocation list.
|
||
//
|
||
|
||
PoolDesc->TotalPages += 1;
|
||
Entry->PoolType = 0;
|
||
Entry->PoolIndex = PACK_POOL_INDEX(PoolIndex);
|
||
|
||
//
|
||
// N.B. A byte is used to store the block size in units of the
|
||
// smallest block size. Therefore, if the number of small
|
||
// blocks in the page is greater than 255, the block size
|
||
// is set to 255.
|
||
//
|
||
|
||
if ((PAGE_SIZE / POOL_SMALLEST_BLOCK) > 255) {
|
||
Entry->BlockSize = 255;
|
||
|
||
} else {
|
||
Entry->BlockSize = PAGE_SIZE / POOL_SMALLEST_BLOCK;
|
||
}
|
||
|
||
Entry->PreviousSize = 0;
|
||
ListHead = &PoolDesc->ListHeads[POOL_LIST_HEADS - 1];
|
||
InsertHeadList(ListHead, ((PLIST_ENTRY)((PCHAR)Entry + POOL_OVERHEAD)));
|
||
} while (TRUE);
|
||
}
|
||
}
|
||
|
||
USHORT
|
||
ExpInsertPoolTracker (
|
||
ULONG Key,
|
||
ULONG Size,
|
||
POOL_TYPE PoolType
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function insert a pool tag in the tag table and increments the
|
||
number of allocates and updates the total allocation size.
|
||
|
||
Arguments:
|
||
|
||
Key - Supplies the key value used to locate a matching entry in the
|
||
tag table.
|
||
|
||
Size - Supplies the allocation size.
|
||
|
||
PoolType - Supplies the pool type.
|
||
|
||
Return Value:
|
||
|
||
The tag index is returned as the function value.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
USHORT Result;
|
||
ULONG Hash;
|
||
ULONG Index;
|
||
KIRQL OldIrql;
|
||
|
||
//
|
||
// Ignore protected pool bit except for returned hash index
|
||
//
|
||
|
||
if (Key & PROTECTED_POOL) {
|
||
Key &= ~PROTECTED_POOL;
|
||
Result = (USHORT)(PROTECTED_POOL >> 16);
|
||
} else {
|
||
Result = 0;
|
||
}
|
||
|
||
if (Key == PoolHitTag) {
|
||
DbgBreakPoint();
|
||
}
|
||
|
||
//
|
||
// Compute hash index and search for pool tag.
|
||
//
|
||
|
||
Hash = ((40543*((((((((PUCHAR)&Key)[0]<<2)^((PUCHAR)&Key)[1])<<2)^((PUCHAR)&Key)[2])<<2)^((PUCHAR)&Key)[3]))>>2) & TRACKER_TABLE_MASK;
|
||
Index = Hash;
|
||
ExAcquireSpinLock(&ExpTaggedPoolLock, &OldIrql);
|
||
do {
|
||
if ((PoolTrackTable[Hash].Key == Key) ||
|
||
(PoolTrackTable[Hash].Key == 0)) {
|
||
goto EntryFound;
|
||
}
|
||
|
||
Hash = (Hash + 1) & TRACKER_TABLE_MASK;
|
||
} while (Hash != Index);
|
||
|
||
//
|
||
// No matching entry and no free entry was found.
|
||
//
|
||
|
||
Hash = MAX_TRACKER_TABLE - 1;
|
||
|
||
//
|
||
// Update pool tracker table entry.
|
||
//
|
||
|
||
EntryFound:
|
||
PoolTrackTable[Hash].Key = Key;
|
||
if ((PoolType & BASE_POOL_TYPE_MASK) == PagedPool) {
|
||
PoolTrackTable[Hash].PagedAllocs += 1;
|
||
PoolTrackTable[Hash].PagedBytes += Size;
|
||
|
||
} else {
|
||
PoolTrackTable[Hash].NonPagedAllocs += 1;
|
||
PoolTrackTable[Hash].NonPagedBytes += Size;
|
||
}
|
||
|
||
ExReleaseSpinLock(&ExpTaggedPoolLock, OldIrql);
|
||
return (USHORT)Hash | Result;
|
||
}
|
||
|
||
|
||
VOID
|
||
ExpRemovePoolTracker (
|
||
ULONG Key,
|
||
ULONG Size,
|
||
POOL_TYPE PoolType
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function increments the number of frees and updates the total
|
||
allocation size.
|
||
|
||
Arguments:
|
||
|
||
Key - Supplies the key value used to locate a matching entry in the
|
||
tag table.
|
||
|
||
Size - Supplies the allocation size.
|
||
|
||
PoolType - Supplies the pool type.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
ULONG Hash;
|
||
ULONG Index;
|
||
KIRQL OldIrql;
|
||
|
||
//
|
||
// Ignore protected pool bit
|
||
//
|
||
|
||
Key &= ~PROTECTED_POOL;
|
||
if (Key == PoolHitTag) {
|
||
DbgBreakPoint();
|
||
}
|
||
|
||
//
|
||
// Compute hash index and search for pool tag.
|
||
//
|
||
|
||
Hash = ((40543*((((((((PUCHAR)&Key)[0]<<2)^((PUCHAR)&Key)[1])<<2)^((PUCHAR)&Key)[2])<<2)^((PUCHAR)&Key)[3]))>>2) & TRACKER_TABLE_MASK;
|
||
Index = Hash;
|
||
ExAcquireSpinLock(&ExpTaggedPoolLock, &OldIrql);
|
||
do {
|
||
if (PoolTrackTable[Hash].Key == Key) {
|
||
goto EntryFound;
|
||
}
|
||
|
||
if (PoolTrackTable[Hash].Key == 0) {
|
||
KdPrint(("POOL: Unable to find tracker %lx, table corrupted\n", Key));
|
||
goto ExitRoutine;
|
||
}
|
||
|
||
|
||
Hash = (Hash + 1) & TRACKER_TABLE_MASK;
|
||
} while (Hash != Index);
|
||
|
||
//
|
||
// No matching entry and no free entry was found.
|
||
//
|
||
|
||
Hash = MAX_TRACKER_TABLE - 1;
|
||
|
||
//
|
||
// Update pool tracker table entry.
|
||
//
|
||
|
||
EntryFound:
|
||
if ((PoolType & BASE_POOL_TYPE_MASK) == PagedPool) {
|
||
PoolTrackTable[Hash].PagedBytes -= Size;
|
||
PoolTrackTable[Hash].PagedFrees += 1;
|
||
|
||
} else {
|
||
PoolTrackTable[Hash].NonPagedBytes -= Size;
|
||
PoolTrackTable[Hash].NonPagedFrees += 1;
|
||
}
|
||
|
||
ExitRoutine:
|
||
ExReleaseSpinLock(&ExpTaggedPoolLock, OldIrql);
|
||
return;
|
||
}
|
||
|
||
|
||
PPOOL_TRACKER_BIG_PAGES
|
||
ExpAddTagForBigPages (
|
||
IN PVOID Va,
|
||
IN ULONG Key
|
||
)
|
||
{
|
||
ULONG Hash;
|
||
BOOLEAN Inserted = TRUE;
|
||
KIRQL OldIrql;
|
||
|
||
Hash = ((ULONG)Va >> PAGE_SHIFT) & BIGPAGE_TABLE_MASK;
|
||
ExAcquireSpinLock(&ExpTaggedPoolLock, &OldIrql);
|
||
while (PoolBigPageTable[Hash].Va != NULL) {
|
||
Hash += 1;
|
||
if (Hash > MAX_BIGPAGE_TABLE) {
|
||
if (!Inserted) {
|
||
if (!FirstPrint) {
|
||
KdPrint(("POOL:unable to insert big page slot %lx\n",Key));
|
||
FirstPrint = TRUE;
|
||
}
|
||
|
||
ExReleaseSpinLock(&ExpTaggedPoolLock, OldIrql);
|
||
return NULL;
|
||
}
|
||
|
||
Hash = 0;
|
||
Inserted = FALSE;
|
||
}
|
||
}
|
||
|
||
PoolBigPageTable[Hash].Va = Va;
|
||
PoolBigPageTable[Hash].Key = Key;
|
||
ExReleaseSpinLock(&ExpTaggedPoolLock, OldIrql);
|
||
return &PoolBigPageTable[Hash];
|
||
}
|
||
|
||
|
||
ULONG
|
||
ExpFindAndRemoveTagBigPages (
|
||
IN PVOID Va
|
||
)
|
||
|
||
{
|
||
|
||
ULONG Hash;
|
||
BOOLEAN Inserted = TRUE;
|
||
KIRQL OldIrql;
|
||
ULONG ReturnKey;
|
||
|
||
Hash = ((ULONG)Va >> PAGE_SHIFT) & BIGPAGE_TABLE_MASK;
|
||
ExAcquireSpinLock(&ExpTaggedPoolLock, &OldIrql);
|
||
while (PoolBigPageTable[Hash].Va != Va) {
|
||
Hash += 1;
|
||
if (Hash > MAX_BIGPAGE_TABLE) {
|
||
if (!Inserted) {
|
||
if (!FirstPrint) {
|
||
KdPrint(("POOL:unable to find big page slot %lx\n",Va));
|
||
FirstPrint = TRUE;
|
||
}
|
||
|
||
ExReleaseSpinLock(&ExpTaggedPoolLock, OldIrql);
|
||
return ' GIB';
|
||
}
|
||
|
||
Hash = 0;
|
||
Inserted = FALSE;
|
||
}
|
||
}
|
||
|
||
PoolBigPageTable[Hash].Va = NULL;
|
||
ReturnKey = PoolBigPageTable[Hash].Key;
|
||
ExReleaseSpinLock(&ExpTaggedPoolLock, OldIrql);
|
||
return ReturnKey;
|
||
}
|
||
|
||
|
||
ULONG
|
||
ExpAllocatePoolWithQuotaHandler(
|
||
IN NTSTATUS ExceptionCode,
|
||
IN PVOID PoolAddress,
|
||
IN BOOLEAN ContinueSearch
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function is called when an exception occurs in ExFreePool
|
||
while quota is being charged to a process.
|
||
|
||
Its function is to deallocate the pool block and continue the search
|
||
for an exception handler.
|
||
|
||
Arguments:
|
||
|
||
ExceptionCode - Supplies the exception code that caused this
|
||
function to be entered.
|
||
|
||
PoolAddress - Supplies the address of a pool block that needs to be
|
||
deallocated.
|
||
|
||
ContinueSearch - Supplies a value that if TRUE causes the exception
|
||
search to continue. This is used in allocate pool with quota
|
||
calls that do not contain the pool quota mask bit set.
|
||
|
||
Return Value:
|
||
|
||
EXCEPTION_CONTINUE_SEARCH - The exception should be propagated to the
|
||
caller of ExAllocatePoolWithQuota.
|
||
|
||
--*/
|
||
|
||
{
|
||
if ( PoolAddress ) {
|
||
ASSERT(ExceptionCode == STATUS_QUOTA_EXCEEDED);
|
||
ExFreePool(PoolAddress);
|
||
|
||
} else {
|
||
ASSERT(ExceptionCode == STATUS_INSUFFICIENT_RESOURCES);
|
||
}
|
||
|
||
return ContinueSearch ? EXCEPTION_CONTINUE_SEARCH : EXCEPTION_EXECUTE_HANDLER;
|
||
}
|
||
|
||
PVOID
|
||
ExAllocatePoolWithQuota(
|
||
IN POOL_TYPE PoolType,
|
||
IN ULONG NumberOfBytes
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function allocates a block of pool of the specified type,
|
||
returns a pointer to the allocated block, and if the binary buddy
|
||
allocator was used to satisfy the request, charges pool quota to the
|
||
current process. This function is used to access both the
|
||
page-aligned pools, and the binary buddy.
|
||
|
||
If the number of bytes specifies a size that is too large to be
|
||
satisfied by the appropriate binary buddy pool, then the
|
||
page-aligned pool allocator is used. The allocated block will be
|
||
page-aligned and a page-sized multiple. No quota is charged to the
|
||
current process if this is the case.
|
||
|
||
Otherwise, the appropriate binary buddy pool is used. The allocated
|
||
block will be 64-bit aligned, but will not be page aligned. After
|
||
the allocation completes, an attempt will be made to charge pool
|
||
quota (of the appropriate type) to the current process object. If
|
||
the quota charge succeeds, then the pool block's header is adjusted
|
||
to point to the current process. The process object is not
|
||
dereferenced until the pool is deallocated and the appropriate
|
||
amount of quota is returned to the process. Otherwise, the pool is
|
||
deallocated, a "quota exceeded" condition is raised.
|
||
|
||
Arguments:
|
||
|
||
PoolType - Supplies the type of pool to allocate. If the pool type
|
||
is one of the "MustSucceed" pool types and sufficient quota
|
||
exists, then this call will always succeed and return a pointer
|
||
to allocated pool. Otherwise, if the system can not allocate
|
||
the requested amount of memory a STATUS_INSUFFICIENT_RESOURCES
|
||
status is raised.
|
||
|
||
NumberOfBytes - Supplies the number of bytes to allocate.
|
||
|
||
Return Value:
|
||
|
||
NON-NULL - Returns a pointer to the allocated pool.
|
||
|
||
Unspecified - If insuffient quota exists to complete the pool
|
||
allocation, the return value is unspecified.
|
||
|
||
--*/
|
||
|
||
{
|
||
return (ExAllocatePoolWithQuotaTag (PoolType, NumberOfBytes, 'enoN'));
|
||
}
|
||
|
||
|
||
PVOID
|
||
ExAllocatePoolWithQuotaTag(
|
||
IN POOL_TYPE PoolType,
|
||
IN ULONG NumberOfBytes,
|
||
IN ULONG Tag
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function allocates a block of pool of the specified type,
|
||
returns a pointer to the allocated block, and if the binary buddy
|
||
allocator was used to satisfy the request, charges pool quota to the
|
||
current process. This function is used to access both the
|
||
page-aligned pools, and the binary buddy.
|
||
|
||
If the number of bytes specifies a size that is too large to be
|
||
satisfied by the appropriate binary buddy pool, then the
|
||
page-aligned pool allocator is used. The allocated block will be
|
||
page-aligned and a page-sized multiple. No quota is charged to the
|
||
current process if this is the case.
|
||
|
||
Otherwise, the appropriate binary buddy pool is used. The allocated
|
||
block will be 64-bit aligned, but will not be page aligned. After
|
||
the allocation completes, an attempt will be made to charge pool
|
||
quota (of the appropriate type) to the current process object. If
|
||
the quota charge succeeds, then the pool block's header is adjusted
|
||
to point to the current process. The process object is not
|
||
dereferenced until the pool is deallocated and the appropriate
|
||
amount of quota is returned to the process. Otherwise, the pool is
|
||
deallocated, a "quota exceeded" condition is raised.
|
||
|
||
Arguments:
|
||
|
||
PoolType - Supplies the type of pool to allocate. If the pool type
|
||
is one of the "MustSucceed" pool types and sufficient quota
|
||
exists, then this call will always succeed and return a pointer
|
||
to allocated pool. Otherwise, if the system can not allocate
|
||
the requested amount of memory a STATUS_INSUFFICIENT_RESOURCES
|
||
status is raised.
|
||
|
||
NumberOfBytes - Supplies the number of bytes to allocate.
|
||
|
||
Return Value:
|
||
|
||
NON-NULL - Returns a pointer to the allocated pool.
|
||
|
||
Unspecified - If insuffient quota exists to complete the pool
|
||
allocation, the return value is unspecified.
|
||
|
||
--*/
|
||
|
||
{
|
||
PVOID p;
|
||
PEPROCESS Process;
|
||
PPOOL_HEADER Entry;
|
||
BOOLEAN IgnoreQuota = FALSE;
|
||
BOOLEAN RaiseOnQuotaFailure = TRUE;
|
||
|
||
if ( PoolType & POOL_QUOTA_FAIL_INSTEAD_OF_RAISE ) {
|
||
RaiseOnQuotaFailure = FALSE;
|
||
PoolType &= ~POOL_QUOTA_FAIL_INSTEAD_OF_RAISE;
|
||
}
|
||
|
||
if (PoolTrackTable
|
||
#if i386 && !FPO
|
||
|| (NtGlobalFlag & FLG_KERNEL_STACK_TRACE_DB)
|
||
#endif // i386 && !FPO
|
||
) {
|
||
IgnoreQuota = TRUE;
|
||
} else {
|
||
PoolType = (POOL_TYPE)((UCHAR)PoolType + POOL_QUOTA_MASK);
|
||
}
|
||
|
||
p = ExAllocatePoolWithTag(PoolType, NumberOfBytes, Tag);
|
||
|
||
//
|
||
// Note - NULL is page aligned.
|
||
//
|
||
|
||
if (!PAGE_ALIGNED(p) && !IgnoreQuota ) {
|
||
|
||
#if POOL_CACHE_SUPPORTED
|
||
|
||
//
|
||
// Align entry on pool allocation boundary.
|
||
//
|
||
|
||
if (((ULONG)p & POOL_CACHE_CHECK) == 0) {
|
||
Entry = (PPOOL_HEADER)((ULONG)p - PoolCacheSize);
|
||
} else {
|
||
Entry = (PPOOL_HEADER)((PCH)p - POOL_OVERHEAD);
|
||
}
|
||
|
||
#else
|
||
Entry = (PPOOL_HEADER)((PCH)p - POOL_OVERHEAD);
|
||
#endif //POOL_CACHE_SUPPORTED
|
||
|
||
Process = PsGetCurrentProcess();
|
||
|
||
//
|
||
// Catch exception and back out allocation if necessary
|
||
//
|
||
|
||
try {
|
||
|
||
Entry->ProcessBilled = NULL;
|
||
|
||
if ( Process != PsInitialSystemProcess ) {
|
||
PsChargePoolQuota(Process,
|
||
PoolType & BASE_POOL_TYPE_MASK,
|
||
(ULONG)(Entry->BlockSize << POOL_BLOCK_SHIFT));
|
||
|
||
ObReferenceObject(Process);
|
||
Entry->ProcessBilled = Process;
|
||
}
|
||
|
||
} except ( ExpAllocatePoolWithQuotaHandler(GetExceptionCode(),p,RaiseOnQuotaFailure)) {
|
||
if ( RaiseOnQuotaFailure ) {
|
||
KeBugCheck(GetExceptionCode());
|
||
}
|
||
else {
|
||
p = NULL;
|
||
}
|
||
}
|
||
|
||
} else {
|
||
if ( !p && RaiseOnQuotaFailure ) {
|
||
ExRaiseStatus(STATUS_INSUFFICIENT_RESOURCES);
|
||
}
|
||
}
|
||
|
||
return p;
|
||
}
|
||
|
||
VOID
|
||
ExFreePool(
|
||
IN PVOID P
|
||
)
|
||
{
|
||
#ifdef POOL_TAGGING
|
||
ExFreePoolWithTag(P, 0);
|
||
return;
|
||
}
|
||
|
||
VOID
|
||
ExFreePoolWithTag(
|
||
IN PVOID P,
|
||
IN ULONG TagToFree
|
||
)
|
||
{
|
||
#else
|
||
ULONG TagToFree = 0;
|
||
#endif
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function deallocates a block of pool. This function is used to
|
||
deallocate to both the page aligned pools, and the buddy (less than
|
||
a page) pools.
|
||
|
||
If the address of the block being deallocated is page-aligned, then
|
||
the page-aliged pool deallocator is used.
|
||
|
||
Otherwise, the binary buddy pool deallocator is used. Deallocation
|
||
looks at the allocated block's pool header to determine the pool
|
||
type and block size being deallocated. If the pool was allocated
|
||
using ExAllocatePoolWithQuota, then after the deallocation is
|
||
complete, the appropriate process's pool quota is adjusted to reflect
|
||
the deallocation, and the process object is dereferenced.
|
||
|
||
Arguments:
|
||
|
||
P - Supplies the address of the block of pool being deallocated.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
--*/
|
||
|
||
POOL_TYPE CheckType;
|
||
PPOOL_HEADER Entry;
|
||
ULONG Index;
|
||
KIRQL LockHandle;
|
||
PSMALL_POOL_LOOKASIDE LookasideList;
|
||
PPOOL_HEADER NextEntry;
|
||
ULONG PoolIndex;
|
||
POOL_TYPE PoolType;
|
||
PPOOL_DESCRIPTOR PoolDesc;
|
||
PEPROCESS ProcessBilled = NULL;
|
||
PKPRCB Prcb;
|
||
BOOLEAN Combined;
|
||
ULONG BigPages;
|
||
ULONG Tag;
|
||
|
||
#if DBG
|
||
BOOLEAN
|
||
ExpCheckForResource(
|
||
IN PVOID p,
|
||
IN ULONG Size
|
||
);
|
||
#endif //DBG
|
||
|
||
#if DBG
|
||
if ((P > MmSpecialPoolStart) && (P < MmSpecialPoolEnd)) {
|
||
MmFreeSpecialPool (P);
|
||
return;
|
||
}
|
||
#endif //DBG
|
||
|
||
//
|
||
// If entry is page aligned, then call free block to the page aligned
|
||
// pool. Otherwise, free the block to the allocation lists.
|
||
//
|
||
|
||
if (PAGE_ALIGNED(P)) {
|
||
|
||
PoolType = MmDeterminePoolType(P);
|
||
CheckType = PoolType & BASE_POOL_TYPE_MASK;
|
||
PoolDesc = PoolVector[PoolType];
|
||
|
||
LOCK_POOL(PoolDesc, LockHandle);
|
||
|
||
PoolDesc->RunningDeAllocs++;
|
||
BigPages = MiFreePoolPages(P);
|
||
PoolDesc->TotalBigPages -= BigPages;
|
||
|
||
#if DBG
|
||
|
||
//
|
||
// Check is an ERESOURCE is current active in this memory block.
|
||
//
|
||
|
||
ExpCheckForResource(P, BigPages * PAGE_SIZE);
|
||
|
||
#endif // DBG
|
||
|
||
UNLOCK_POOL(PoolDesc, LockHandle);
|
||
|
||
if (PoolTrackTable != NULL) {
|
||
Tag = ExpFindAndRemoveTagBigPages(P);
|
||
if (Tag & PROTECTED_POOL) {
|
||
Tag &= ~PROTECTED_POOL;
|
||
TagToFree &= ~PROTECTED_POOL;
|
||
if (Tag != TagToFree) {
|
||
DbgPrint( "EX: Invalid attempt to free protected pool block %x (%c%c%c%c)\n",
|
||
P,
|
||
Tag,
|
||
Tag >> 8,
|
||
Tag >> 16,
|
||
Tag >> 24
|
||
);
|
||
DbgBreakPoint();
|
||
}
|
||
}
|
||
|
||
ExpRemovePoolTracker(Tag,
|
||
BigPages * PAGE_SIZE,
|
||
PoolType);
|
||
}
|
||
|
||
#if DBG
|
||
|
||
if (ExpEchoPoolCalls){
|
||
PVOID CallingAddress;
|
||
PVOID CallersCaller;
|
||
|
||
DbgPrint("0x%lx EXDEALLOC: from %s", P, PoolTypeNames[PoolType]);
|
||
RtlGetCallersAddress(&CallingAddress, &CallersCaller);
|
||
DbgPrint(" Callers:%lx, %lx\n", CallingAddress, CallersCaller);
|
||
}
|
||
|
||
#endif //DBG
|
||
|
||
} else {
|
||
|
||
//
|
||
// Align the entry address to a pool allocation boundary.
|
||
//
|
||
|
||
#if POOL_CACHE_SUPPORTED
|
||
|
||
if (((ULONG)P & POOL_CACHE_CHECK) == 0) {
|
||
Entry = (PPOOL_HEADER)((ULONG)P - PoolCacheSize);
|
||
|
||
} else {
|
||
Entry = (PPOOL_HEADER)((PCHAR)P - POOL_OVERHEAD);
|
||
}
|
||
|
||
#else
|
||
|
||
Entry = (PPOOL_HEADER)((PCHAR)P - POOL_OVERHEAD);
|
||
|
||
#endif //POOL_CACHE_SUPPORTED
|
||
|
||
PoolType = (Entry->PoolType & POOL_TYPE_MASK) - 1;
|
||
CheckType = PoolType & BASE_POOL_TYPE_MASK;
|
||
|
||
#if DBG
|
||
|
||
//
|
||
// Check if an ERESOURCE is currently active in this memory block.
|
||
//
|
||
|
||
ExpCheckForResource(Entry, (ULONG)(Entry->BlockSize << POOL_BLOCK_SHIFT));
|
||
|
||
//
|
||
// Check if the pool type field is defined correctly.
|
||
//
|
||
|
||
if (Entry->PoolType == 0) {
|
||
DbgPrint("EX: Invalid pool header 0x%lx 0x%lx\n",P,*(PULONG)P);
|
||
KeBugCheckEx(BAD_POOL_HEADER, 1, (ULONG)Entry, *(PULONG)Entry, 0);
|
||
}
|
||
|
||
//
|
||
// Check if the pool index field is defined correctly.
|
||
//
|
||
|
||
if ((CheckType == NonPagedPool) && (Entry->PoolIndex != 0)) {
|
||
DbgPrint("EX: Invalid pool header 0x%lx 0x%lx\n",Entry,*(PULONG)Entry);
|
||
KeBugCheckEx(BAD_POOL_HEADER, 2, (ULONG)Entry, *(PULONG)Entry, 0);
|
||
|
||
} else if (((CheckType == PagedPool) && (Entry->PoolIndex == 0)) ||
|
||
(((Entry->PoolIndex >> 4) & 0xf) != (Entry->PoolIndex & 0xf))) {
|
||
DbgPrint("EX: Invalid pool header 0x%lx 0x%lx\n",Entry,*(PULONG)Entry);
|
||
KeBugCheckEx(BAD_POOL_HEADER, 4, (ULONG)Entry, *(PULONG)Entry, 0);
|
||
}
|
||
|
||
#endif // DBG
|
||
|
||
if (Entry->PoolType & POOL_QUOTA_MASK) {
|
||
if (PoolTrackTable == NULL) {
|
||
ProcessBilled = Entry->ProcessBilled;
|
||
Entry->PoolTag = 'atoQ';
|
||
}
|
||
}
|
||
|
||
#if DBG
|
||
|
||
if (ExpEchoPoolCalls){
|
||
PVOID CallingAddress;
|
||
PVOID CallersCaller;
|
||
|
||
DbgPrint("0x%lx EXDEALLOC: from %s", P, PoolTypeNames[PoolType]);
|
||
RtlGetCallersAddress(&CallingAddress, &CallersCaller);
|
||
DbgPrint(" Callers:%lx, %lx\n", CallingAddress, CallersCaller);
|
||
}
|
||
|
||
#endif
|
||
|
||
#ifdef CHECK_POOL_TAIL
|
||
|
||
if (NtGlobalFlag & FLG_POOL_ENABLE_TAIL_CHECK) {
|
||
PCHAR PoolBlock;
|
||
ULONG CountBytes;
|
||
ULONG CountBytesEqual;
|
||
|
||
PoolBlock = (PCHAR)(((PPOOL_BLOCK)Entry + Entry->BlockSize)) - POOL_SMALLEST_BLOCK;
|
||
CountBytes = POOL_SMALLEST_BLOCK;
|
||
CountBytesEqual = RtlCompareMemoryUlong(PoolBlock,
|
||
CountBytes,
|
||
ALLOCATED_POOL);
|
||
|
||
if (CountBytesEqual != CountBytes) {
|
||
DbgPrint("EX: Pool block at %lx modified at %lx past requested size of %lx\n",
|
||
PoolBlock,
|
||
PoolBlock + CountBytesEqual,
|
||
(Entry->BlockSize << POOL_BLOCK_SHIFT) - POOL_SMALLEST_BLOCK - POOL_OVERHEAD);
|
||
|
||
DbgBreakPoint();
|
||
}
|
||
}
|
||
|
||
#endif //CHECK_POOL_TAIL
|
||
|
||
#if i386 && !FPO
|
||
if ((NtGlobalFlag & FLG_KERNEL_STACK_TRACE_DB) &&
|
||
PoolTrackTable != NULL &&
|
||
Entry->AllocatorBackTraceIndex != 0 &&
|
||
Entry->AllocatorBackTraceIndex & POOL_BACKTRACEINDEX_PRESENT) {
|
||
if (Entry->PoolTagHash & (PROTECTED_POOL >> 16)) {
|
||
Entry->PoolTagHash &= ~(PROTECTED_POOL >> 16);
|
||
Tag = PROTECTED_POOL;
|
||
} else {
|
||
Tag = 0;
|
||
}
|
||
Entry->PoolTag = Tag | PoolTrackTable[Entry->PoolTagHash].Key;
|
||
}
|
||
#endif // i386 && !FPO
|
||
|
||
//
|
||
// If pool tagging is enabled, then update the pool tracking database.
|
||
// Otherwise, check to determine if quota was charged when the pool
|
||
// block was allocated.
|
||
//
|
||
|
||
if (PoolTrackTable != NULL) {
|
||
Tag = Entry->PoolTag;
|
||
if (Tag & PROTECTED_POOL) {
|
||
Tag &= ~PROTECTED_POOL;
|
||
TagToFree &= ~PROTECTED_POOL;
|
||
if (Tag != TagToFree) {
|
||
DbgPrint( "EX: Invalid attempt to free protected pool block %x (%c%c%c%c)\n",
|
||
P,
|
||
Tag,
|
||
Tag >> 8,
|
||
Tag >> 16,
|
||
Tag >> 24
|
||
);
|
||
DbgBreakPoint();
|
||
}
|
||
}
|
||
ExpRemovePoolTracker(Tag,
|
||
Entry->BlockSize << POOL_BLOCK_SHIFT ,
|
||
PoolType);
|
||
|
||
} else if (ProcessBilled != NULL) {
|
||
PsReturnPoolQuota(ProcessBilled,
|
||
PoolType & BASE_POOL_TYPE_MASK,
|
||
(ULONG)Entry->BlockSize << POOL_BLOCK_SHIFT);
|
||
ObDereferenceObject(ProcessBilled);
|
||
}
|
||
|
||
//
|
||
// If the pool block is a small block, then attempt to free the block
|
||
// to the single entry lookaside list. If the free atempts fails, then
|
||
// free the block by merging it back into the pool data structures.
|
||
//
|
||
|
||
PoolIndex = UNPACK_POOL_INDEX(Entry->PoolIndex);
|
||
PoolDesc = PoolVector[PoolType];
|
||
Index = Entry->BlockSize;
|
||
if (Index > POOL_SMALL_LISTS) {
|
||
Index = (Index >> SHIFT_OFFSET) + POOL_SMALL_LISTS + 1;
|
||
|
||
} else {
|
||
|
||
//
|
||
// Attempt to free the small block to the per rpocessor single
|
||
// entry lookaside list.
|
||
//
|
||
|
||
#if !defined(CHECK_POOL_TAIL) && (DEADBEEF == 0)
|
||
|
||
Prcb = KeGetCurrentPrcb();
|
||
if (CheckType == PagedPool) {
|
||
|
||
#if defined(_PPC_)
|
||
|
||
if ((Entry = (PPOOL_HEADER)InterlockedExchange((PLONG)&Prcb->PagedFreeEntry[Index - 1],
|
||
(LONG)Entry)) == NULL) {
|
||
|
||
#else
|
||
|
||
LookasideList = &ExpSmallPagedPoolLookasideLists[Index - 1];
|
||
LookasideList->TotalFrees += 1;
|
||
if ((Isx86FeaturePresent(KF_CMPXCHG8B)) &&
|
||
(ExQueryDepthSList(&LookasideList->SListHead) < LookasideList->Depth)) {
|
||
LookasideList->FreeHits += 1;
|
||
Entry += 1;
|
||
ExInterlockedPushEntrySList(&LookasideList->SListHead,
|
||
(PSINGLE_LIST_ENTRY)Entry,
|
||
&LookasideList->Lock);
|
||
|
||
#endif
|
||
|
||
return;
|
||
}
|
||
|
||
PoolIndex = UNPACK_POOL_INDEX(Entry->PoolIndex);
|
||
|
||
} else {
|
||
LookasideList = &ExpSmallNPagedPoolLookasideLists[Index - 1];
|
||
LookasideList->TotalFrees += 1;
|
||
if (ExQueryDepthSList(&LookasideList->SListHead) < LookasideList->Depth) {
|
||
LookasideList->FreeHits += 1;
|
||
Entry += 1;
|
||
ExInterlockedPushEntrySList(&LookasideList->SListHead,
|
||
(PSINGLE_LIST_ENTRY)Entry,
|
||
&LookasideList->Lock);
|
||
|
||
return;
|
||
}
|
||
|
||
PoolIndex = UNPACK_POOL_INDEX(Entry->PoolIndex);
|
||
}
|
||
#endif
|
||
|
||
}
|
||
|
||
//
|
||
// If the pool type is paged pool, then get the appropriate pool
|
||
// descriptor address.
|
||
//
|
||
|
||
if (CheckType == PagedPool) {
|
||
PoolDesc = &PoolDesc[PoolIndex];
|
||
}
|
||
|
||
ASSERT(PoolIndex == PoolDesc->PoolIndex);
|
||
|
||
LOCK_POOL(PoolDesc, LockHandle);
|
||
|
||
#if DEADBEEF
|
||
if (NtGlobalFlag & FLG_POOL_ENABLE_FREE_CHECK) {
|
||
CHECK_POOL_PAGE(Entry);
|
||
}
|
||
#endif
|
||
|
||
PoolDesc->RunningDeAllocs += 1;
|
||
|
||
#if DEADBEEF
|
||
if (NtGlobalFlag & FLG_POOL_ENABLE_FREE_CHECK) {
|
||
|
||
//
|
||
// Fill in the block with its previous pool tags.
|
||
//
|
||
|
||
ExFillFreedPool( (PCHAR)Entry + POOL_OVERHEAD,
|
||
(Entry->BlockSize << POOL_BLOCK_SHIFT) - POOL_OVERHEAD,
|
||
Entry->PoolTag );
|
||
}
|
||
#endif
|
||
|
||
//
|
||
// Free the specified pool block.
|
||
//
|
||
// Check to see if the next entry is free.
|
||
//
|
||
|
||
Combined = FALSE;
|
||
NextEntry = (PPOOL_HEADER)((PPOOL_BLOCK)Entry + Entry->BlockSize);
|
||
if (PAGE_END(NextEntry) == FALSE) {
|
||
if (NextEntry->PoolType == 0) {
|
||
|
||
//
|
||
// This block is free, combine with the released block.
|
||
//
|
||
|
||
CHECK_LIST(__LINE__, ((PLIST_ENTRY)((PCHAR)NextEntry + POOL_OVERHEAD)), P);
|
||
|
||
Combined = TRUE;
|
||
RemoveEntryList(((PLIST_ENTRY)((PCHAR)NextEntry + POOL_OVERHEAD)));
|
||
Entry->BlockSize += NextEntry->BlockSize;
|
||
|
||
#if DEADBEEF
|
||
if (NtGlobalFlag & FLG_POOL_ENABLE_FREE_CHECK) {
|
||
|
||
//
|
||
// Overwrite the poolheader with its tag because we're being combined
|
||
// an the previous block.
|
||
//
|
||
|
||
ExFillFreedPool( (PCHAR)NextEntry,
|
||
POOL_OVERHEAD + sizeof(LIST_ENTRY),
|
||
NextEntry->PoolTag );
|
||
}
|
||
#endif
|
||
}
|
||
}
|
||
|
||
//
|
||
// Check to see if the previous entry is free.
|
||
//
|
||
|
||
if (Entry->PreviousSize != 0) {
|
||
NextEntry = (PPOOL_HEADER)((PPOOL_BLOCK)Entry - Entry->PreviousSize);
|
||
if (NextEntry->PoolType == 0) {
|
||
|
||
//
|
||
// This block is free, combine with the released block.
|
||
//
|
||
|
||
CHECK_LIST(__LINE__, ((PLIST_ENTRY)((PCHAR)NextEntry + POOL_OVERHEAD)), P);
|
||
|
||
Combined = TRUE;
|
||
RemoveEntryList(((PLIST_ENTRY)((PCHAR)NextEntry + POOL_OVERHEAD)));
|
||
NextEntry->BlockSize += Entry->BlockSize;
|
||
|
||
#if DEADBEEF
|
||
if (NtGlobalFlag & FLG_POOL_ENABLE_FREE_CHECK) {
|
||
|
||
//
|
||
// Overwrite the poolheader with its tag because we're being combined
|
||
// an the previous block.
|
||
//
|
||
|
||
ExFillFreedPool( (PCHAR)Entry,
|
||
POOL_OVERHEAD + sizeof(LIST_ENTRY),
|
||
Entry->PoolTag );
|
||
}
|
||
#endif
|
||
|
||
Entry = NextEntry;
|
||
}
|
||
}
|
||
|
||
//
|
||
// If the block being freed has been combined into a full page,
|
||
// then return the free the page to memory management.
|
||
//
|
||
|
||
if (PAGE_ALIGNED(Entry) &&
|
||
(PAGE_END((PPOOL_BLOCK)Entry + Entry->BlockSize) != FALSE)) {
|
||
|
||
//
|
||
// If the pool type is paged pool, then the paged pool page lock
|
||
// must be held during the free of the pool pages.
|
||
//
|
||
|
||
LOCK_IF_PAGED_POOL(CheckType);
|
||
|
||
MiFreePoolPages(Entry);
|
||
|
||
UNLOCK_IF_PAGED_POOL(CheckType);
|
||
|
||
PoolDesc->TotalPages -= 1;
|
||
|
||
} else {
|
||
|
||
//
|
||
// Insert this element into the list.
|
||
//
|
||
|
||
Entry->PoolType = 0;
|
||
Entry->PoolIndex = PACK_POOL_INDEX(PoolIndex);
|
||
Index = Entry->BlockSize;
|
||
if (Index > POOL_SMALL_LISTS) {
|
||
Index = (Index >> SHIFT_OFFSET) + POOL_SMALL_LISTS + 1;
|
||
}
|
||
|
||
//
|
||
// If the freed block was combined with any other block, then
|
||
// adjust the size of the next block if necessary.
|
||
//
|
||
|
||
if (Combined != FALSE) {
|
||
|
||
//
|
||
// The size of this entry has changed, if this entry is
|
||
// not the last one in the page, update the pool block
|
||
// after this block to have a new previous allocation size.
|
||
//
|
||
|
||
NextEntry = (PPOOL_HEADER)((PPOOL_BLOCK)Entry + Entry->BlockSize);
|
||
if (PAGE_END(NextEntry) == FALSE) {
|
||
NextEntry->PreviousSize = Entry->BlockSize;
|
||
}
|
||
|
||
//
|
||
// Reduce fragmentation and insert at the tail in hopes
|
||
// neighbors for this will be freed before this is reallocated.
|
||
//
|
||
|
||
CHECK_LIST(__LINE__, &PoolDesc->ListHeads[Index - 1], P);
|
||
|
||
InsertTailList(&PoolDesc->ListHeads[Index - 1],
|
||
((PLIST_ENTRY)((PCHAR)Entry + POOL_OVERHEAD)));
|
||
|
||
} else {
|
||
|
||
CHECK_LIST(__LINE__, &PoolDesc->ListHeads[Index - 1], P);
|
||
|
||
InsertHeadList(&PoolDesc->ListHeads[Index - 1],
|
||
((PLIST_ENTRY)((PCHAR)Entry + POOL_OVERHEAD)));
|
||
}
|
||
}
|
||
|
||
UNLOCK_POOL(PoolDesc, LockHandle);
|
||
|
||
}
|
||
|
||
return;
|
||
}
|
||
|
||
|
||
ULONG
|
||
ExQueryPoolBlockSize (
|
||
IN PVOID PoolBlock,
|
||
OUT PBOOLEAN QuotaCharged
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function returns the size of the pool block.
|
||
|
||
Arguments:
|
||
|
||
PoolBlock - Supplies the address of the block of pool.
|
||
|
||
QuotaCharged - Supplies a BOOLEAN variable to receive whether or not the
|
||
pool block had quota charged.
|
||
|
||
NOTE: If the entry is bigger than a page, the value PAGE_SIZE is returned
|
||
rather than the correct number of bytes.
|
||
|
||
Return Value:
|
||
|
||
Size of pool block.
|
||
|
||
--*/
|
||
|
||
{
|
||
PPOOL_HEADER Entry;
|
||
ULONG size;
|
||
|
||
if (PAGE_ALIGNED(PoolBlock)) {
|
||
*QuotaCharged = FALSE;
|
||
return PAGE_SIZE;
|
||
}
|
||
|
||
#if POOL_CACHE_SUPPORTED
|
||
|
||
//
|
||
// Align entry on pool allocation boundary.
|
||
//
|
||
|
||
if (((ULONG)PoolBlock & POOL_CACHE_CHECK) == 0) {
|
||
Entry = (PPOOL_HEADER)((ULONG)PoolBlock - PoolCacheSize);
|
||
size = (Entry->BlockSize << POOL_BLOCK_SHIFT) - PoolCacheSize;
|
||
|
||
} else {
|
||
Entry = (PPOOL_HEADER)((PCHAR)PoolBlock - POOL_OVERHEAD);
|
||
size = (Entry->BlockSize << POOL_BLOCK_SHIFT) - POOL_OVERHEAD;
|
||
}
|
||
|
||
#else
|
||
|
||
Entry = (PPOOL_HEADER)((PCHAR)PoolBlock - POOL_OVERHEAD);
|
||
size = (Entry->BlockSize << POOL_BLOCK_SHIFT) - POOL_OVERHEAD;
|
||
|
||
#endif //POOL_CACHE_SUPPORTED
|
||
|
||
#ifdef CHECK_POOL_TAIL
|
||
|
||
size = size - POOL_SMALLEST_BLOCK;
|
||
|
||
#endif
|
||
|
||
if ( PoolTrackTable ) {
|
||
*QuotaCharged = FALSE;
|
||
}
|
||
else {
|
||
*QuotaCharged = (BOOLEAN) (Entry->ProcessBilled != NULL);
|
||
}
|
||
return size;
|
||
}
|
||
|
||
VOID
|
||
ExQueryPoolUsage(
|
||
OUT PULONG PagedPoolPages,
|
||
OUT PULONG NonPagedPoolPages,
|
||
OUT PULONG PagedPoolAllocs,
|
||
OUT PULONG PagedPoolFrees,
|
||
OUT PULONG PagedPoolLookasideHits,
|
||
OUT PULONG NonPagedPoolAllocs,
|
||
OUT PULONG NonPagedPoolFrees,
|
||
OUT PULONG NonPagedPoolLookasideHits
|
||
)
|
||
|
||
{
|
||
|
||
ULONG Count;
|
||
ULONG Index;
|
||
PPOOL_DESCRIPTOR pd;
|
||
PKPRCB Prcb;
|
||
|
||
//
|
||
// Sum all the paged pool usage.
|
||
//
|
||
|
||
pd = PoolVector[PagedPool];
|
||
*PagedPoolPages = 0;
|
||
*PagedPoolAllocs = 0;
|
||
*PagedPoolFrees = 0;
|
||
|
||
for (Index = 0; Index < ExpNumberOfPagedPools + 1; Index += 1) {
|
||
*PagedPoolPages += pd[Index].TotalPages + pd[Index].TotalBigPages;
|
||
*PagedPoolAllocs += pd[Index].RunningAllocs;
|
||
*PagedPoolFrees += pd[Index].RunningDeAllocs;
|
||
}
|
||
|
||
//
|
||
// Sum all the nonpaged pool usage.
|
||
//
|
||
|
||
pd = PoolVector[NonPagedPool];
|
||
*NonPagedPoolPages = pd->TotalPages + pd->TotalBigPages;
|
||
*NonPagedPoolAllocs = pd->RunningAllocs;
|
||
*NonPagedPoolFrees = pd->RunningDeAllocs;
|
||
|
||
//
|
||
// Sum all the nonpaged must succeed usage.
|
||
//
|
||
|
||
pd = PoolVector[NonPagedPoolMustSucceed];
|
||
*NonPagedPoolPages += pd->TotalPages + pd->TotalBigPages;
|
||
*NonPagedPoolAllocs += pd->RunningAllocs;
|
||
*NonPagedPoolFrees += pd->RunningDeAllocs;
|
||
|
||
//
|
||
// Sum all the lookaside hits for paged and nonpaged pool.
|
||
//
|
||
|
||
for (Index = 0; Index < (ULONG)KeNumberProcessors; Index += 1) {
|
||
Prcb = KiProcessorBlock[Index];
|
||
if (Prcb != NULL) {
|
||
|
||
#if defined(_PPC_)
|
||
|
||
*PagedPoolLookasideHits += Prcb->PagedPoolLookasideHits;
|
||
for (Count = 0; Count < POOL_SMALL_LISTS; Count +=1) {
|
||
*NonPagedPoolLookasideHits += ExpSmallNPagedPoolLookasideLists[Count].AllocateHits;
|
||
}
|
||
|
||
#else
|
||
|
||
for (Count = 0; Count < POOL_SMALL_LISTS; Count +=1) {
|
||
*PagedPoolLookasideHits += ExpSmallPagedPoolLookasideLists[Count].AllocateHits;
|
||
*NonPagedPoolLookasideHits += ExpSmallNPagedPoolLookasideLists[Count].AllocateHits;
|
||
}
|
||
|
||
#endif
|
||
|
||
}
|
||
}
|
||
|
||
return;
|
||
}
|
||
|
||
VOID
|
||
ExReturnPoolQuota(
|
||
IN PVOID P
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function returns quota charged to a subject process when the
|
||
specified pool block was allocated.
|
||
|
||
Arguments:
|
||
|
||
P - Supplies the address of the block of pool being deallocated.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
PPOOL_HEADER Entry;
|
||
POOL_TYPE PoolType;
|
||
PEPROCESS Process;
|
||
|
||
//
|
||
// Align the entry address to a pool allocation boundary.
|
||
//
|
||
|
||
#if POOL_CACHE_SUPPORTED
|
||
|
||
if (((ULONG)P & POOL_CACHE_CHECK) == 0) {
|
||
Entry = (PPOOL_HEADER)((ULONG)P - PoolCacheSize);
|
||
|
||
} else {
|
||
Entry = (PPOOL_HEADER)((PCHAR)P - POOL_OVERHEAD);
|
||
}
|
||
|
||
#else
|
||
|
||
Entry = (PPOOL_HEADER)((PCHAR)P - POOL_OVERHEAD);
|
||
|
||
#endif //POOL_CACHE_SUPPORTED
|
||
|
||
//
|
||
// If quota was charged, then return the appropriate quota to the
|
||
// subject process.
|
||
//
|
||
|
||
PoolType = (Entry->PoolType & POOL_TYPE_MASK) - 1;
|
||
if ((Entry->PoolType & POOL_QUOTA_MASK) &&
|
||
(PoolTrackTable == NULL)) {
|
||
Process = Entry->ProcessBilled;
|
||
Entry->PoolTag = 'atoQ';
|
||
Entry->PoolType &= ~POOL_QUOTA_MASK;
|
||
if (Process != NULL) {
|
||
PsReturnPoolQuota(Process,
|
||
PoolType & BASE_POOL_TYPE_MASK,
|
||
(ULONG)Entry->BlockSize << POOL_BLOCK_SHIFT);
|
||
|
||
ObDereferenceObject(Process);
|
||
}
|
||
}
|
||
|
||
return;
|
||
}
|
||
|
||
#if DBG || (i386 && !FPO)
|
||
|
||
//
|
||
// Only works on checked builds or free x86 builds with FPO turned off
|
||
// See comment in mm\allocpag.c
|
||
//
|
||
|
||
NTSTATUS
|
||
ExpSnapShotPoolPages(
|
||
IN PVOID Address,
|
||
IN ULONG Size,
|
||
IN OUT PSYSTEM_POOL_INFORMATION PoolInformation,
|
||
IN OUT PSYSTEM_POOL_ENTRY *PoolEntryInfo,
|
||
IN ULONG Length,
|
||
IN OUT PULONG RequiredLength
|
||
)
|
||
{
|
||
NTSTATUS Status;
|
||
CLONG i;
|
||
PPOOL_HEADER p;
|
||
|
||
if (PAGE_ALIGNED(Address)) {
|
||
for (i = 0; i < MAX_BIGPAGE_TABLE; i++) {
|
||
if (PoolBigPageTable[i].NumberOfPages != 0 &&
|
||
PoolBigPageTable[i].Va == Address
|
||
) {
|
||
PoolInformation->NumberOfEntries += 1;
|
||
*RequiredLength += sizeof(SYSTEM_POOL_ENTRY);
|
||
if (Length < *RequiredLength) {
|
||
Status = STATUS_INFO_LENGTH_MISMATCH;
|
||
|
||
} else {
|
||
(*PoolEntryInfo)->Allocated = TRUE;
|
||
(*PoolEntryInfo)->Size = (PoolBigPageTable[i].NumberOfPages * PAGE_SIZE);
|
||
(*PoolEntryInfo)->AllocatorBackTraceIndex = 0;
|
||
#if i386 && !FPO
|
||
if ((NtGlobalFlag & FLG_KERNEL_STACK_TRACE_DB)) {
|
||
(*PoolEntryInfo)->AllocatorBackTraceIndex =
|
||
PoolBigPageTable[i].AllocatorBackTraceIndex;
|
||
}
|
||
#endif // i386 && !FPO
|
||
(*PoolEntryInfo)->ProcessChargedQuota = 0;
|
||
#if !DBG
|
||
if (NtGlobalFlag & FLG_POOL_ENABLE_TAGGING)
|
||
#endif //!DBG
|
||
(*PoolEntryInfo)->TagUlong = PoolBigPageTable[i].Key;
|
||
(*PoolEntryInfo)++;
|
||
Status = STATUS_SUCCESS;
|
||
}
|
||
|
||
return Status;
|
||
}
|
||
}
|
||
}
|
||
|
||
p = (PPOOL_HEADER)Address;
|
||
if ((Size == PAGE_SIZE) && (p->PreviousSize == 0)) {
|
||
ULONG EntrySize;
|
||
|
||
do {
|
||
EntrySize = p->BlockSize << POOL_BLOCK_SHIFT;
|
||
if (EntrySize == 0) {
|
||
return STATUS_COMMITMENT_LIMIT;
|
||
}
|
||
PoolInformation->NumberOfEntries += 1;
|
||
*RequiredLength += sizeof(SYSTEM_POOL_ENTRY);
|
||
if (Length < *RequiredLength) {
|
||
Status = STATUS_INFO_LENGTH_MISMATCH;
|
||
|
||
} else {
|
||
(*PoolEntryInfo)->Size = EntrySize;
|
||
if (p->PoolType != 0) {
|
||
(*PoolEntryInfo)->Allocated = TRUE;
|
||
(*PoolEntryInfo)->AllocatorBackTraceIndex = 0;
|
||
(*PoolEntryInfo)->ProcessChargedQuota = 0;
|
||
#if !DBG
|
||
if (NtGlobalFlag & FLG_POOL_ENABLE_TAGGING)
|
||
#endif //!DBG
|
||
(*PoolEntryInfo)->TagUlong = p->PoolTag;
|
||
#if i386 && !FPO
|
||
if ((NtGlobalFlag & FLG_KERNEL_STACK_TRACE_DB) &&
|
||
p->AllocatorBackTraceIndex != 0 &&
|
||
p->AllocatorBackTraceIndex & POOL_BACKTRACEINDEX_PRESENT
|
||
) {
|
||
(*PoolEntryInfo)->AllocatorBackTraceIndex = p->AllocatorBackTraceIndex ^ POOL_BACKTRACEINDEX_PRESENT;
|
||
#if !DBG
|
||
if (NtGlobalFlag & FLG_POOL_ENABLE_TAGGING) {
|
||
#else
|
||
if (TRUE) {
|
||
#endif //!DBG
|
||
if (p->PoolTagHash & (PROTECTED_POOL >> 16)) {
|
||
(*PoolEntryInfo)->TagUlong = PROTECTED_POOL;
|
||
} else {
|
||
(*PoolEntryInfo)->TagUlong = 0;
|
||
}
|
||
(*PoolEntryInfo)->TagUlong |= PoolTrackTable[p->PoolTagHash&~(PROTECTED_POOL >> 16)].Key;
|
||
}
|
||
}
|
||
#endif // i386 && !FPO
|
||
|
||
} else {
|
||
(*PoolEntryInfo)->Allocated = FALSE;
|
||
(*PoolEntryInfo)->AllocatorBackTraceIndex = 0;
|
||
(*PoolEntryInfo)->ProcessChargedQuota = 0;
|
||
#if !DBG
|
||
if (NtGlobalFlag & FLG_POOL_ENABLE_TAGGING)
|
||
#endif //!DBG
|
||
(*PoolEntryInfo)->TagUlong = p->PoolTag;
|
||
}
|
||
|
||
(*PoolEntryInfo)++;
|
||
Status = STATUS_SUCCESS;
|
||
}
|
||
|
||
p = (PPOOL_HEADER)((PCHAR)p + EntrySize);
|
||
}
|
||
while (PAGE_END(p) == FALSE);
|
||
|
||
} else {
|
||
PoolInformation->NumberOfEntries += 1;
|
||
*RequiredLength += sizeof(SYSTEM_POOL_ENTRY);
|
||
if (Length < *RequiredLength) {
|
||
Status = STATUS_INFO_LENGTH_MISMATCH;
|
||
|
||
} else {
|
||
(*PoolEntryInfo)->Allocated = TRUE;
|
||
(*PoolEntryInfo)->Size = Size;
|
||
(*PoolEntryInfo)->AllocatorBackTraceIndex = 0;
|
||
(*PoolEntryInfo)->ProcessChargedQuota = 0;
|
||
(*PoolEntryInfo)++;
|
||
Status = STATUS_SUCCESS;
|
||
}
|
||
}
|
||
|
||
return Status;
|
||
}
|
||
|
||
NTSTATUS
|
||
ExSnapShotPool(
|
||
IN POOL_TYPE PoolType,
|
||
IN PSYSTEM_POOL_INFORMATION PoolInformation,
|
||
IN ULONG Length,
|
||
OUT PULONG ReturnLength OPTIONAL
|
||
)
|
||
|
||
{
|
||
|
||
ULONG Index;
|
||
PVOID Lock;
|
||
KIRQL LockHandle;
|
||
PPOOL_DESCRIPTOR PoolDesc;
|
||
ULONG RequiredLength;
|
||
NTSTATUS Status;
|
||
|
||
RequiredLength = FIELD_OFFSET(SYSTEM_POOL_INFORMATION, Entries);
|
||
if (Length < RequiredLength) {
|
||
return STATUS_INFO_LENGTH_MISMATCH;
|
||
}
|
||
|
||
try {
|
||
|
||
//
|
||
// If the pool type is paged, then lock all of the paged pools.
|
||
// Otherwise, lock the nonpaged pool.
|
||
//
|
||
|
||
PoolDesc = PoolVector[PoolType];
|
||
if (PoolType == PagedPool) {
|
||
Index = 0;
|
||
KeRaiseIrql(APC_LEVEL, &LockHandle); \
|
||
do {
|
||
Lock = PoolDesc[Index].LockAddress;
|
||
ExAcquireFastMutex((PFAST_MUTEX)Lock);
|
||
Index += 1;
|
||
} while (Index < ExpNumberOfPagedPools);
|
||
|
||
} else {
|
||
ExAcquireSpinLock(&NonPagedPoolLock, &LockHandle);
|
||
}
|
||
|
||
PoolInformation->EntryOverhead = POOL_OVERHEAD;
|
||
PoolInformation->NumberOfEntries = 0;
|
||
|
||
#if POOL_CACHE_SUPPORTED //only compile for machines which have a nonzero value.
|
||
|
||
if (PoolType & CACHE_ALIGNED_POOL_TYPE_MASK) {
|
||
PoolInformation->EntryOverhead = (USHORT)PoolCacheSize;
|
||
}
|
||
|
||
#endif //POOL_CACHE_SUPPORTED
|
||
|
||
Status = MmSnapShotPool(PoolType,
|
||
ExpSnapShotPoolPages,
|
||
PoolInformation,
|
||
Length,
|
||
&RequiredLength);
|
||
|
||
} finally {
|
||
|
||
//
|
||
// If the pool type is paged, then unlock all of the paged pools.
|
||
// Otherwise, unlock the nonpaged pool.
|
||
//
|
||
|
||
if (PoolType == PagedPool) {
|
||
Index = 0;
|
||
do {
|
||
Lock = PoolDesc[Index].LockAddress;
|
||
ExReleaseFastMutex((PFAST_MUTEX)Lock);
|
||
Index += 1;
|
||
} while (Index < ExpNumberOfPagedPools);
|
||
|
||
KeLowerIrql(LockHandle);
|
||
|
||
} else {
|
||
ExReleaseSpinLock(&NonPagedPoolLock, LockHandle);
|
||
}
|
||
}
|
||
|
||
if (ARGUMENT_PRESENT(ReturnLength)) {
|
||
*ReturnLength = RequiredLength;
|
||
}
|
||
|
||
return Status;
|
||
}
|
||
#endif // DBG || (i386 && !FPO)
|
||
|
||
#if i386 && !FPO
|
||
USHORT
|
||
ExGetPoolBackTraceIndex(
|
||
IN PVOID P
|
||
)
|
||
{
|
||
CLONG i;
|
||
PPOOL_HEADER Entry;
|
||
|
||
if (NtGlobalFlag & FLG_KERNEL_STACK_TRACE_DB) {
|
||
if ( PAGE_ALIGNED(P) ) {
|
||
for (i = 0; i < MAX_BIGPAGE_TABLE; i++) {
|
||
if (PoolBigPageTable[i].NumberOfPages != 0 &&
|
||
PoolBigPageTable[i].Va == P
|
||
) {
|
||
return PoolBigPageTable[i].AllocatorBackTraceIndex;
|
||
}
|
||
}
|
||
|
||
} else {
|
||
#if POOL_CACHE_SUPPORTED
|
||
|
||
//
|
||
// Align entry on pool allocation boundary.
|
||
//
|
||
|
||
if (((ULONG)P & POOL_CACHE_CHECK) == 0) {
|
||
Entry = (PPOOL_HEADER)((ULONG)P - PoolCacheSize);
|
||
} else {
|
||
Entry = (PPOOL_HEADER)((PCH)P - POOL_OVERHEAD);
|
||
}
|
||
|
||
#else
|
||
Entry = (PPOOL_HEADER)((PCH)P - POOL_OVERHEAD);
|
||
#endif //POOL_CACHE_SUPPORTED
|
||
|
||
if (Entry->AllocatorBackTraceIndex != 0 &&
|
||
Entry->AllocatorBackTraceIndex & POOL_BACKTRACEINDEX_PRESENT
|
||
) {
|
||
return (Entry->AllocatorBackTraceIndex ^ POOL_BACKTRACEINDEX_PRESENT);
|
||
}
|
||
}
|
||
}
|
||
|
||
return 0;
|
||
}
|
||
#endif // i386 && !FPO
|