1565 lines
46 KiB
C++
1565 lines
46 KiB
C++
//+---------------------------------------------------------------------------
|
|
//
|
|
// Microsoft Windows
|
|
// Copyright (C) Microsoft Corporation, 1991 - 2000.
|
|
//
|
|
// File: NORM.CXX
|
|
//
|
|
// Contents: Normalizer
|
|
//
|
|
// Classes: CNormalizer
|
|
//
|
|
// History: 28-May-91 t-WadeR added CNormalizer
|
|
// 31-Jan-92 BartoszM Created from lang.cxx
|
|
// 07-Oct-93 DwightKr Added new methods to normalize
|
|
// different data types
|
|
//
|
|
// Notes: The filtering pipeline is hidden in the Data Repository
|
|
// object which serves as a sink for the filter.
|
|
// The sink for the Data Repository is the Key Repository.
|
|
// The language dependent part of the pipeline
|
|
// is obtained from the Language List object and is called
|
|
// Key Maker. It consists of:
|
|
//
|
|
// Word Breaker
|
|
// Stemmer (optional)
|
|
// Normalizer
|
|
// Noise List
|
|
//
|
|
// Each object serves as a sink for its predecessor,
|
|
// Key Repository is the final sink.
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
|
|
#include <pch.cxx>
|
|
#pragma hdrstop
|
|
|
|
#include <plang.hxx>
|
|
#include <misc.hxx>
|
|
#include <norm.hxx>
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Function GetExpAndSign
|
|
//
|
|
// Synopsis: Finds the exponent and sign of a number
|
|
//
|
|
// Arguments: [d] -- the input number to examine
|
|
// [fPositive] -- returns TRUE if positive, FALSE if negative
|
|
//
|
|
// Returns: The exponent
|
|
//
|
|
// History: 21-Nov-94 KyleP Created.
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
|
|
int GetExpAndSign( double d, BOOL & fPositive )
|
|
{
|
|
//
|
|
// bit 63 = sign
|
|
// bits 52 - 62 = exponent
|
|
// bits 0 - 51 = mantissa
|
|
//
|
|
|
|
Win4Assert( sizeof(LARGE_INTEGER) == sizeof(double) );
|
|
|
|
LARGE_INTEGER * pli = (LARGE_INTEGER *)&d;
|
|
|
|
fPositive = (pli->HighPart & 0x80000000) == 0;
|
|
|
|
int const bias = 0x3ff;
|
|
|
|
return ( ( pli->HighPart & 0x7ff00000 ) >> 20 ) - bias;
|
|
} //GetExpAndSign
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Function NormDouble
|
|
//
|
|
// Synopsis: Normalizes doubles by taking log2 of the number
|
|
//
|
|
// Notes: This func converts doubles into one of 5 different categories
|
|
//
|
|
// x < -1x2**32 is in bin 0
|
|
// -1x2**32 <= x <= -1x2**-32 are in bins 1 to 65
|
|
// -1x2**-32 <= x <= 1x2**-32 is in bin 66
|
|
// 1x2**-32 <= x <= 1x2**32 are in bins 67 to 131
|
|
// x > 1x2**32 is bin bin 132
|
|
//
|
|
// History: 21-Nov-94 KyleP Created.
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
|
|
static unsigned NormDouble(double dValue)
|
|
{
|
|
const int SignificantExponent = 32;
|
|
const int SignificantRange = SignificantExponent * 2;
|
|
|
|
const unsigned LowestBin = 0; // 0
|
|
const unsigned LowerBin = LowestBin + 1; // 1
|
|
const unsigned MiddleBin = LowerBin + SignificantRange + 1; // 66
|
|
const unsigned UpperBin = MiddleBin + 1; // 67
|
|
const unsigned HighestBin = UpperBin+ SignificantRange + 1; // 132
|
|
|
|
|
|
BOOL fPositive;
|
|
|
|
int exp = GetExpAndSign( dValue, fPositive );
|
|
|
|
unsigned bin;
|
|
|
|
if ( exp < -SignificantExponent )
|
|
{
|
|
//
|
|
// All numbers close to zero in middle bin
|
|
//
|
|
|
|
bin = MiddleBin;
|
|
}
|
|
else if ( exp > SignificantExponent )
|
|
{
|
|
if ( fPositive )
|
|
{
|
|
//
|
|
// Very large positive numbers in top bin
|
|
//
|
|
|
|
bin = HighestBin;
|
|
}
|
|
else
|
|
{
|
|
//
|
|
// Very large negative numbers in bottom bin
|
|
//
|
|
|
|
bin = LowestBin;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if ( fPositive )
|
|
{
|
|
//
|
|
// medium size positive numbers
|
|
//
|
|
|
|
bin = UpperBin + exp + SignificantExponent;
|
|
}
|
|
else
|
|
{
|
|
//
|
|
// medium size negative numbers
|
|
//
|
|
|
|
bin = LowerBin - exp + SignificantExponent;
|
|
}
|
|
}
|
|
return bin;
|
|
}
|
|
|
|
#ifdef TEST_NORM
|
|
//
|
|
// a test to verify the validity of the NormDouble function.
|
|
//
|
|
void TestNormDouble()
|
|
{
|
|
float fVal0 = 0.;
|
|
float fVal1 = 1.;
|
|
unsigned nZero = NormDouble( fVal0 );
|
|
unsigned nOne = NormDouble( fVal1 );
|
|
|
|
printf(" Value:Bin %f : 0x%4X (%d)\n", fVal0, nZero, nZero );
|
|
printf(" Value:Bin %f : 0x%4X (%d)\n", fVal1, nOne, nOne );
|
|
|
|
BOOL fPos;
|
|
float f = fVal1;
|
|
unsigned nPrev = nOne;
|
|
while ( f > fVal0 )
|
|
{
|
|
unsigned nVal = NormDouble( f );
|
|
if (nVal > nPrev || nVal < nZero || nVal > nOne)
|
|
{
|
|
printf(" Value:Bin %f : 0x%4X (%d)\tExp %d\n", f, nVal, nVal, GetExpAndSign(f, fPos) );
|
|
}
|
|
|
|
nPrev = nVal;
|
|
f = f/3;
|
|
}
|
|
|
|
f = fVal1;
|
|
nPrev = nOne;
|
|
while ( f < 1e+32 )
|
|
{
|
|
unsigned nVal = NormDouble( f );
|
|
if (nVal < nPrev)
|
|
printf(" Value:Bin %f : 0x%4X (%d)\n", f, nVal, nVal );
|
|
|
|
nPrev = nVal;
|
|
f = f * (float)1.5;
|
|
}
|
|
|
|
float fValm1 = -1.;
|
|
unsigned nMinusOne = NormDouble( fValm1 );
|
|
|
|
printf(" Value:Bin %f : 0x%4X (%d)\n", fValm1, nMinusOne, nMinusOne );
|
|
|
|
f = fValm1;
|
|
nPrev = nMinusOne;
|
|
while ( f < fVal0 )
|
|
{
|
|
unsigned nVal = NormDouble( f );
|
|
if (nVal < nPrev || nVal > nZero || nVal < nMinusOne)
|
|
printf(" Value:Bin %f : 0x%4X (%d)\tExp %d\n", f, nVal, nVal, GetExpAndSign(f, fPos) );
|
|
|
|
nPrev = nVal;
|
|
f = f/3;
|
|
}
|
|
|
|
f = fValm1;
|
|
nPrev = nMinusOne;
|
|
while ( f > -1e+32 )
|
|
{
|
|
unsigned nVal = NormDouble( f );
|
|
if (nVal > nPrev)
|
|
printf(" Value:Bin %f : 0x%4X (%d)\n", f, nVal, nVal );
|
|
|
|
nPrev = nVal;
|
|
f = f * (float)1.5;
|
|
}
|
|
}
|
|
#endif // 0
|
|
|
|
|
|
// ------------------------------------------------------------------------
|
|
// | Upper Limit | Divisor (2^x) | # of Bins | (in hex) |
|
|
// ------------------------------------------------------------------------
|
|
// | 2^10 - 1 | 2^0 | 2^10 - 0 | 0400 - 0000 |
|
|
// | 2^16 - 1 | 2^3 | 2^12 - 2^7 | 2000 - 0080 |
|
|
// | 2^20 - 1 | 2^6 | 2^14 - 2^10 | 4000 - 0400 |
|
|
// | 2^26 - 1 | 2^13 | 2^13 - 2^7 | 2000 - 0080 |
|
|
// | 2^30 - 1 | 2^23 | 2^7 - 2^3 | 0080 - 0008 |
|
|
// | 2^31 - 1 | 2^25 | 2^6 - 2^5 | 0040 - 0020 |
|
|
// ------------------------------------------------------------------------
|
|
// | Total | | | 84C0 - 04D8 |
|
|
// | | | | 7FE8 |
|
|
// ------------------------------------------------------------------------
|
|
|
|
const long limit1 = 0x400;
|
|
const long shift1 = 0;
|
|
const long cbins1 = 0x400;
|
|
|
|
const long limit2 = 0x10000; // 2^16
|
|
const long shift2 = 3;
|
|
const long cSkip1 = limit1 >> shift2;
|
|
const long cbins2 = (limit2 >> shift2)-cSkip1;
|
|
|
|
const long limit3 = 0x100000; // 2^20
|
|
const long shift3 = 6;
|
|
const long cSkip2 = limit2 >> shift3;
|
|
const long cbins3 = (limit3 >> shift3) - cSkip2;
|
|
|
|
const long limit4 = 0x4000000; // 2^26
|
|
const long shift4 = 13;
|
|
const long cSkip3 = limit3 >> shift4;
|
|
const long cbins4 = (limit4 >> shift4) - cSkip3;
|
|
|
|
const long limit5 = 0x40000000; // 2^30
|
|
const long shift5 = 23;
|
|
const long cSkip4 = limit4 >> shift5;
|
|
const long cbins5 = (limit5 >> shift5) - cSkip4;
|
|
|
|
const long limit6 = MINLONG; // 2^31
|
|
const long shift6 = 25;
|
|
const long cSkip5 = limit5 >> shift6;
|
|
const long cbins6 = ((long) ((unsigned) limit6 >> shift6)) - cSkip5;
|
|
|
|
static unsigned MapLong( LONG lValue )
|
|
{
|
|
|
|
Win4Assert( !(lValue & MINLONG) || ( MINLONG == lValue ) );
|
|
|
|
#if CIDBG==1
|
|
const long cTotal = cbins1 + cbins2 + cbins3 + cbins4 + cbins5 + cbins6;
|
|
Win4Assert( cTotal <= MINSHORT );
|
|
#endif // CIDBG == 1
|
|
|
|
unsigned ulValue = (unsigned) lValue;
|
|
|
|
unsigned binNum = (unsigned) lValue;;
|
|
|
|
if ( ulValue < limit1 )
|
|
{
|
|
//
|
|
// Nothing to do.
|
|
//
|
|
}
|
|
else if ( ulValue < limit2 )
|
|
{
|
|
binNum = cbins1 - cSkip1 + (ulValue >> shift2);
|
|
}
|
|
else if ( ulValue < limit3 )
|
|
{
|
|
binNum = cbins1 + cbins2 - cSkip2 + (binNum >> shift3);
|
|
}
|
|
else if ( ulValue < limit4 )
|
|
{
|
|
binNum = cbins1 + cbins2 + cbins3 - cSkip3 + (binNum >> shift4);
|
|
}
|
|
else if ( ulValue < limit5 )
|
|
{
|
|
binNum = cbins1 + cbins2 + cbins3 + cbins4 - cSkip4 + (binNum >> shift5);
|
|
}
|
|
else
|
|
{
|
|
binNum = cbins1 + cbins2 + cbins3 + cbins4 + cbins5 - cSkip5 + (binNum >> shift6);
|
|
}
|
|
|
|
return binNum;
|
|
}
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Function: NormLong
|
|
//
|
|
// Synopsis: Normalizes the given "signed" long value to a value between
|
|
// 0x0000 - 0xFFFF. The negative numbers occupy 0x0000-0x8000.
|
|
// Positive numbers occupy 0x8000-0xFFFF
|
|
//
|
|
// Arguments: [lValue] - The value to be normalized.
|
|
//
|
|
// History: 10-03-95 srikants Created
|
|
//
|
|
// Notes:
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
|
|
static unsigned NormLong(LONG lValue)
|
|
{
|
|
if (lValue >= 0)
|
|
{
|
|
return MapLong(lValue) + MINSHORT;
|
|
}
|
|
else
|
|
{
|
|
return MINSHORT - MapLong(-lValue);
|
|
}
|
|
}
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Function: NormULong
|
|
//
|
|
// Synopsis: Normalizes an "unsigned" long value to a value between
|
|
// 0x0000-0xFFFF. Numbers from 0-2^31 - 1 are mapped in the
|
|
// range 0x0000-0x7FFF. Numbers 2^31 to 2^32 - 1 are mapped
|
|
// in the range 0x8000 - 0xFFFF
|
|
//
|
|
// Arguments: [lValue] - The value to be mapped.
|
|
//
|
|
// History: 10-03-95 srikants Created
|
|
//
|
|
// Notes:
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
|
|
static unsigned NormULong( ULONG lValue )
|
|
{
|
|
unsigned val = MapLong( lValue & ~MINLONG ); // turn off the high bit
|
|
|
|
Win4Assert( !(val & MINSHORT) );
|
|
|
|
if ( lValue & MINLONG )
|
|
val |= MINSHORT;
|
|
|
|
return val;
|
|
}
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Function: MapLargeInteger
|
|
//
|
|
// Synopsis: Maps a LargeInteger to a number between 0x0000-0x7FFF.
|
|
//
|
|
// Numbers with the "HighPart" = 0 are mapped in the range
|
|
// 0x0000-0x3FFF. When the HighPart !=0, the values are
|
|
// mapped to 0x4000 - 0x7FFF
|
|
//
|
|
// Arguments: [liValue] - The value to be mapped.
|
|
//
|
|
// History: 10-03-95 srikants Created
|
|
//
|
|
// Notes:
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
|
|
static unsigned MapLargeInteger( LARGE_INTEGER & liValue )
|
|
{
|
|
Win4Assert( !(liValue.HighPart & MINLONG) || ( MINLONG == liValue.HighPart ) );
|
|
|
|
unsigned normVal;
|
|
|
|
if ( 0 == liValue.HighPart )
|
|
{
|
|
normVal = NormULong( liValue.LowPart );
|
|
normVal >>= 2;
|
|
}
|
|
else
|
|
{
|
|
normVal = MapLong( liValue.HighPart ); // 0x0000-0x7FFF
|
|
normVal >>= 1;
|
|
normVal |= 0x4000;
|
|
}
|
|
|
|
Win4Assert( normVal < 0x8000 );
|
|
|
|
return normVal;
|
|
}
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Function: NormULargeInteger
|
|
//
|
|
// Synopsis: Normalizes an unsigned LargeInteger to a number between
|
|
// 0x0000-0xFFFF.
|
|
//
|
|
// Numbers with the "HighPart" = 0 are mapped in the range
|
|
// 0x0000-0x7FFF. When the HighPart !=0, the values are
|
|
// mapped to 0x8000 - 0xFFFF.
|
|
//
|
|
// Arguments: [uliValue] - The value to be mapped.
|
|
//
|
|
// History: 02-09-96 Alanw Created
|
|
//
|
|
// Notes:
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
|
|
static unsigned NormULargeInteger( ULARGE_INTEGER & uliValue )
|
|
{
|
|
unsigned normVal;
|
|
|
|
if ( 0 == uliValue.HighPart )
|
|
{
|
|
normVal = NormULong( uliValue.LowPart );
|
|
normVal >>= 1;
|
|
}
|
|
else
|
|
{
|
|
normVal = NormULong( uliValue.HighPart ); // 0x0000-0x7FFF
|
|
normVal |= 0x8000;
|
|
}
|
|
|
|
Win4Assert( normVal < 0x10000 );
|
|
|
|
return normVal;
|
|
}
|
|
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Function: NormLargeInteger
|
|
//
|
|
// Synopsis: Normalizes a large integer to a value between 0x0000-0xFFFF.
|
|
//
|
|
// -ve Numbers are mapped in the range 0x0000-0x8000.
|
|
// +ve numbers are mapped in the range 0x8000-0xFFFF.
|
|
//
|
|
// Arguments: [liValue] - The value to be normalized. Note that the
|
|
// argument is NOT passed by reference. The value is changed
|
|
// in this method and so should not be passed by reference.
|
|
//
|
|
// History: 10-03-95 srikants Created
|
|
//
|
|
// Notes:
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
|
|
static unsigned NormLargeInteger( LARGE_INTEGER liValue )
|
|
{
|
|
unsigned normVal;
|
|
|
|
if ( liValue.QuadPart < 0 )
|
|
{
|
|
liValue.QuadPart = -liValue.QuadPart;
|
|
normVal = MINSHORT - MapLargeInteger( liValue );
|
|
}
|
|
else
|
|
{
|
|
normVal = MINSHORT + MapLargeInteger( liValue );
|
|
}
|
|
|
|
Win4Assert( normVal < 0x10000 );
|
|
|
|
return normVal;
|
|
}
|
|
|
|
#ifdef TEST_NORM
|
|
//
|
|
// a test to verify the validity of the NormLong function.
|
|
//
|
|
void TestNormLong()
|
|
{
|
|
long lVal1 = 0;
|
|
unsigned nVal1 = NormLong( lVal1 );
|
|
|
|
printf(" Value:Bin 0x%8X : 0x%4X \t(%10d : %10d)\n", lVal1, nVal1, lVal1, nVal1 );
|
|
|
|
lVal1 = 2;
|
|
long lVal2 = 0;
|
|
unsigned nVal2 = NormLong(1);
|
|
|
|
while ( !(lVal1 & 0x80000000) )
|
|
{
|
|
nVal1 = NormLong( lVal1 );
|
|
//printf(" Value:Bin 0x%8X : 0x%4X \t(%10d : %10d)\n", lVal1, nVal1, lVal1, nVal1 );
|
|
|
|
Win4Assert( nVal1 == nVal2+1 );
|
|
|
|
lVal2 = lVal1 + lVal1-1;
|
|
nVal2 = NormLong( lVal2 );
|
|
//printf(" Value:Bin 0x%8X : 0x%4X \t(%10d : %10d)\n", lVal2, nVal2, lVal2, nVal2 );
|
|
|
|
lVal1 <<= 1;
|
|
}
|
|
|
|
lVal1 = 2;
|
|
nVal2 = NormLong(-1);
|
|
printf(" Value:Bin 0x%8X : 0x%4X \t(%10d : %10d)\n", -1, nVal2, -1, nVal2 );
|
|
|
|
while ( !(lVal1 & 0x80000000) )
|
|
{
|
|
nVal1 = NormLong( -lVal1 );
|
|
//printf(" Value:Bin 0x%8X : 0x%4X \t(%10d : %10d)\n", -lVal1, nVal1, -lVal1, nVal1 );
|
|
|
|
Win4Assert( nVal1 == nVal2-1 );
|
|
|
|
lVal2 = lVal1 + lVal1-1;
|
|
lVal2 = -lVal2;
|
|
|
|
nVal2 = NormLong( lVal2 );
|
|
//printf(" Value:Bin 0x%8X : 0x%4X \t(%10d : %10d)\n", lVal2, nVal2, lVal2, nVal2 );
|
|
|
|
lVal1 <<= 1;
|
|
}
|
|
}
|
|
#endif // 0
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CNormalizer::CNormalizer
|
|
//
|
|
// Synopsis: constructor for normalizer
|
|
//
|
|
// Effects: gets buffers from noiselist
|
|
//
|
|
// Arguments: [nl] -- Noise list object to pass data on to.
|
|
//
|
|
// History: 05-June-91 t-WadeR Created.
|
|
//
|
|
// Notes:
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
CNormalizer::CNormalizer( PNoiseList& nl )
|
|
: _noiseList(nl)
|
|
{
|
|
SetWordBuffer();
|
|
|
|
// check that input size + prefix fits in the output buffer
|
|
Win4Assert( cwcMaxKey * sizeof( WCHAR ) + cbKeyPrefix <= *_pcbOutBuf );
|
|
}
|
|
|
|
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CNormalizer::GetFlags
|
|
//
|
|
// Synopsis: Returns address of ranking and range flags
|
|
//
|
|
// Arguments: [ppRange] -- range flag
|
|
// [ppRank] -- rank flag
|
|
//
|
|
// History: 11-Fab-92 BartoszM Created.
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
void CNormalizer::GetFlags ( BOOL** ppRange, CI_RANK** ppRank )
|
|
{
|
|
_noiseList.GetFlags ( ppRange, ppRank );
|
|
}
|
|
|
|
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CNormalizer::ProcessAltWord, public
|
|
//
|
|
// Synopsis: Normalizes a UniCode string, passes it to NoiseList.
|
|
//
|
|
// Effects: Deposits a normalized version [pwcInBuf] in [_pbOutBuf]
|
|
//
|
|
// Arguments: [pwcInBuf] -- input buffer
|
|
// [cwc] -- count of chars in pwcInBuf
|
|
//
|
|
// History: 03-May-95 SitaramR Created.
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
|
|
void CNormalizer::ProcessAltWord( WCHAR const *pwcInBuf, ULONG cwc )
|
|
{
|
|
SetNextAltBuffer();
|
|
|
|
unsigned hash = NormalizeWord( pwcInBuf, cwc );
|
|
SetAltHash( hash );
|
|
}
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CNormalizer::ProcessWord, public
|
|
//
|
|
// Synopsis: Normalizes a UniCode string, passes it to NoiseList.
|
|
//
|
|
// Effects: Deposits a normalized version of [pwcInBuf] in [_pbOutBuf].
|
|
//
|
|
// Arguments: [pwcInBuf] -- input buffer
|
|
// [cwc] -- count of chars in pwcInBuf
|
|
//
|
|
// History: 05-June-91 t-WadeR Created.
|
|
// 13-Oct-92 AmyA Added unicode support
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
|
|
void CNormalizer::ProcessWord( WCHAR const *pwcInBuf, ULONG cwc )
|
|
{
|
|
if ( UsingAltBuffers() )
|
|
SetNextAltBuffer();
|
|
|
|
unsigned hash = NormalizeWord( pwcInBuf, cwc );
|
|
|
|
if ( UsingAltBuffers() )
|
|
{
|
|
SetAltHash( hash );
|
|
ProcessAllWords();
|
|
}
|
|
else
|
|
_noiseList.PutWord( hash );
|
|
}
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CNormalizer::ProcessAllWords, private
|
|
//
|
|
// Synopsis: Removes duplicate alternate words and emits remainder.
|
|
//
|
|
// History: 17-Sep-1999 KyleP Created.
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
|
|
void CNormalizer::ProcessAllWords()
|
|
{
|
|
//
|
|
// Check for duplicate keys. Since the number of alternate forms will always be
|
|
// quite small it's ok to use a O(n^2) algorithm here.
|
|
//
|
|
|
|
unsigned iFinal = 0;
|
|
|
|
for ( unsigned i = 0; i < _cAltKey; i++ )
|
|
{
|
|
//
|
|
// Already marked duplicate?
|
|
//
|
|
|
|
if ( 0 == _aAltKey[i].Count() )
|
|
continue;
|
|
|
|
iFinal = i;
|
|
|
|
for ( unsigned j = i+1; j < _cAltKey; j++ )
|
|
{
|
|
//
|
|
// Remember, Pid is really the hash here.
|
|
//
|
|
|
|
if ( _aAltKey[i].Pid() == _aAltKey[j].Pid() &&
|
|
_aAltKey[i].Count() == _aAltKey[j].Count() &&
|
|
RtlEqualMemory( _aAltKey[i].GetBuf(), _aAltKey[j].GetBuf(), _aAltKey[j].Count() ) )
|
|
{
|
|
ciDebugOut(( DEB_TRACE, "Duplicate keys: %u and %u\n", i, j ));
|
|
_aAltKey[j].SetCount( 0 );
|
|
}
|
|
}
|
|
}
|
|
|
|
//
|
|
// Now transfer any remaining key(s).
|
|
//
|
|
|
|
SetWordBuffer();
|
|
unsigned hash;
|
|
|
|
for ( i = 0; i <= iFinal; i++ )
|
|
{
|
|
//
|
|
// Ignore duplicates
|
|
//
|
|
|
|
if ( 0 == _aAltKey[i].Count() )
|
|
continue;
|
|
|
|
//
|
|
// Copy to the transfer buffer.
|
|
//
|
|
|
|
*_pcbOutBuf = _aAltKey[i].Count();
|
|
RtlCopyMemory( _pbOutBuf, _aAltKey[i].GetBuf(), *_pcbOutBuf );
|
|
hash = _aAltKey[i].Pid();
|
|
|
|
//
|
|
// If this is not the final "PutWord" call, send the data along.
|
|
//
|
|
|
|
if ( i != iFinal )
|
|
_noiseList.PutAltWord( hash );
|
|
}
|
|
|
|
//
|
|
// Put the final word
|
|
//
|
|
|
|
_noiseList.PutWord( hash );
|
|
} //ProcessAllWords
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CNormalizer::NormalizeWord
|
|
//
|
|
// Synopsis: Normalizes a UniCode string
|
|
// Calculates the hash function for normalized string.
|
|
//
|
|
// Arguments: [pwcInBuf] -- input buffer
|
|
// [cwc] -- count of chars in pwcInBuf
|
|
//
|
|
// Returns: unsigned hash value of string
|
|
//
|
|
// History: 03-May-95 SitaramR Created.
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
|
|
unsigned CNormalizer::NormalizeWord( WCHAR const *pwcInBuf, ULONG cwc )
|
|
{
|
|
return NormalizeWord( pwcInBuf, cwc, _pbOutBuf, _pcbOutBuf );
|
|
}
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CNormalizer::NormalizeWord
|
|
//
|
|
// Synopsis: Normalizes a UniCode string
|
|
// Calculates the hash function for normalized string. This
|
|
// function is identical to the other NormalizeWord funtion,
|
|
// except that it puts the outputs int he output parameters
|
|
//
|
|
// Arguments: [pwcInBuf] -- input buffer
|
|
// [cwc] -- count of chars in pwcInBuf
|
|
// [pbOutBuf] -- output buffer.
|
|
// [pcbOutBuf] - pointer to output count of bytes.
|
|
//
|
|
// Returns: unsigned hash value of string
|
|
//
|
|
// History: 03-May-1995 SitaramR Created.
|
|
// 03-Oct-2000 KitmanH Added output parameters
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
|
|
unsigned CNormalizer::NormalizeWord( WCHAR const *pwcInBuf,
|
|
ULONG cwc,
|
|
BYTE *pbOutBuf,
|
|
unsigned *pcbOutBuf )
|
|
{
|
|
// count of bytes needs to take into account STRING_KEY
|
|
|
|
*pcbOutBuf = cwc * sizeof(WCHAR) + cbKeyPrefix;
|
|
|
|
// prefix with the string key identifier
|
|
|
|
*pbOutBuf++ = STRING_KEY;
|
|
|
|
unsigned hash = 0;
|
|
|
|
Win4Assert ( cwc != 0 && cwc <= cwcMaxKey );
|
|
for ( unsigned i = 0; i < cwc; i++ )
|
|
{
|
|
WCHAR c = *pwcInBuf++;
|
|
|
|
// normalize the character to upcase.
|
|
|
|
c = ( c < 'a' ) ? c : ( c <= 'z' ) ? ( c - ('a' - 'A') ) :
|
|
RtlUpcaseUnicodeChar( c );
|
|
|
|
//
|
|
// Store. Do it one byte at a time because the normalized string
|
|
// must be byte compared.
|
|
//
|
|
|
|
*pbOutBuf++ = (BYTE)(c >> 8);
|
|
*pbOutBuf++ = (BYTE)c;
|
|
|
|
// hash
|
|
hash = ( hash << 2 ) + c;
|
|
}
|
|
|
|
return hash;
|
|
}
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CNormalizer::NormalizeWstr - Public
|
|
//
|
|
// Synopsis: Normalizes a UniCode string
|
|
//
|
|
// Arguments: [pwcInBuf] -- input buffer
|
|
// [cwcInBuf] -- count of chars in pwcInBuf
|
|
// [pbOutBuf] -- output buffer.
|
|
// [pcbOutBuf] - pointer to output count of bytes.
|
|
//
|
|
// History: 10-Feb-2000 KitmanH Created
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
|
|
void CNormalizer::NormalizeWStr( WCHAR const *pwcInBuf,
|
|
ULONG cwcInBuf,
|
|
BYTE *pbOutBuf,
|
|
unsigned *pcbOutBuf )
|
|
{
|
|
NormalizeWord( pwcInBuf,
|
|
cwcInBuf,
|
|
pbOutBuf,
|
|
pcbOutBuf );
|
|
}
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CValueNormalizer::CValueNormalizer
|
|
//
|
|
// Synopsis: Constructor
|
|
//
|
|
// Arguments: [krep] -- key repository sink for keys
|
|
//
|
|
// History: 21-Sep-92 BartoszM Created.
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
|
|
CValueNormalizer::CValueNormalizer( PKeyRepository& krep )
|
|
: _krep(krep)
|
|
{
|
|
_krep.GetBuffers( &_pcbOutBuf, &_pbOutBuf, &_pOcc );
|
|
_cbMaxOutBuf = *_pcbOutBuf;
|
|
*_pOcc = 0;
|
|
}
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CValueNormalizer::PutValue, public
|
|
//
|
|
// Synopsis: Store a variant
|
|
//
|
|
// Arguments: [pid] -- property id
|
|
// [occ] -- On input: starting occurrence.
|
|
// On output: next starting occurrence.
|
|
// [var] -- value
|
|
//
|
|
// History: 04-Nov-94 KyleP Created.
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
|
|
void CValueNormalizer::PutValue( PROPID pid,
|
|
OCCURRENCE & occ,
|
|
CStorageVariant const & var )
|
|
{
|
|
*_pOcc = occ;
|
|
|
|
switch ( var.Type() )
|
|
{
|
|
case VT_EMPTY:
|
|
case VT_NULL:
|
|
break;
|
|
|
|
case VT_UI1:
|
|
PutValue( pid, var.GetUI1() );
|
|
break;
|
|
|
|
case VT_I1:
|
|
PutValue( pid, var.GetI1() );
|
|
break;
|
|
|
|
case VT_UI2:
|
|
PutValue( pid, (USHORT) var.GetUI2() );
|
|
break;
|
|
|
|
case VT_I2:
|
|
PutValue( pid, var.GetI2() );
|
|
break;
|
|
|
|
case VT_I4:
|
|
case VT_INT:
|
|
PutValue( pid, var.GetI4() );
|
|
break;
|
|
|
|
case VT_R4:
|
|
PutValue( pid, var.GetR4() );
|
|
break;
|
|
|
|
case VT_R8:
|
|
PutValue( pid, var.GetR8() );
|
|
break;
|
|
|
|
case VT_UI4:
|
|
case VT_UINT:
|
|
PutValue( pid, var.GetUI4() );
|
|
break;
|
|
|
|
case VT_I8:
|
|
PutValue( pid, var.GetI8() );
|
|
break;
|
|
|
|
case VT_UI8:
|
|
PutValue( pid, var.GetUI8() );
|
|
break;
|
|
|
|
case VT_BOOL:
|
|
PutValue( pid, (BYTE) (FALSE != var.GetBOOL()) );
|
|
break;
|
|
|
|
case VT_ERROR:
|
|
PutValue( pid, var.GetERROR() );
|
|
break;
|
|
|
|
case VT_CY:
|
|
PutValue( pid, var.GetCY() );
|
|
break;
|
|
|
|
case VT_DATE:
|
|
PutDate( pid, var.GetDATE() );
|
|
break;
|
|
|
|
case VT_FILETIME:
|
|
PutValue( pid, var.GetFILETIME() );
|
|
break;
|
|
|
|
case VT_CLSID:
|
|
PutValue( pid, *var.GetCLSID() );
|
|
break;
|
|
|
|
// NTRAID#DB-NTBUG9-84589-2000/07/31-dlee Indexing Service data type normalization doesn't handle VT_DECIMAL, VT_VECTOR, or VT_ARRAY.
|
|
|
|
default:
|
|
ciDebugOut(( DEB_IWARN, "Unhandled type %d (%x) sent to normalization\n",
|
|
var.Type(), var.Type() ));
|
|
break;
|
|
}
|
|
|
|
occ = *_pOcc;
|
|
}
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CValueNormalizer::PutValue private
|
|
//
|
|
// Synopsis: Store a unsigned 2 byte value without altering it
|
|
//
|
|
// Arguments: [pid] -- property id
|
|
// [uValue] -- value
|
|
// [bType] -- value type
|
|
//
|
|
// History: 07-Oct-93 DwightKr Created.
|
|
//
|
|
// Notes: This is the principal PutValue method that other PutValue()s
|
|
// will call. Each of the OTHER PutValue()'s sole purpose is
|
|
// to normalize their input data into a 2-byte unsigned value.
|
|
// This version of PutValue() will store the value together
|
|
// with its WID, PID, size, etc. in the CDataRepository object.
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
void CValueNormalizer::PutValue( PROPID pid, unsigned uValue, BYTE bType )
|
|
{
|
|
BYTE* pb = _pbOutBuf;
|
|
|
|
// Store size of entry
|
|
*_pcbOutBuf = sizeof(USHORT) + sizeof(PROPID) + 1;
|
|
|
|
// Store key type
|
|
*pb++ = bType;
|
|
|
|
// store property id
|
|
*pb++ = (BYTE)(pid >> 24);
|
|
*pb++ = (BYTE)(pid >> 16);
|
|
*pb++ = (BYTE)(pid >> 8);
|
|
*pb++ = (BYTE) pid;
|
|
|
|
// Store key
|
|
Win4Assert( uValue < 0x10000 );
|
|
*pb++ = BYTE (uValue >> 8);
|
|
*pb++ = BYTE (uValue);
|
|
|
|
#if CIDBG == 1
|
|
for (unsigned i = 0; i < *_pcbOutBuf; i++ )
|
|
{
|
|
ciDebugOut (( DEB_USER1 | DEB_NOCOMPNAME, "%02x ", _pbOutBuf[i] ));
|
|
}
|
|
ciDebugOut (( DEB_USER1 | DEB_NOCOMPNAME, "\n" ));
|
|
#endif
|
|
|
|
_krep.PutPropId(pid);
|
|
_krep.PutKey();
|
|
(*_pOcc)++;
|
|
}
|
|
|
|
void CValueNormalizer::PutMinValue( PROPID pid, OCCURRENCE & occ, VARENUM Type )
|
|
{
|
|
*_pOcc = occ;
|
|
PutValue( pid, 0, Type );
|
|
occ = *_pOcc;
|
|
}
|
|
|
|
void CValueNormalizer::PutMaxValue( PROPID pid, OCCURRENCE & occ, VARENUM Type )
|
|
{
|
|
*_pOcc = occ;
|
|
PutValue( pid, 0xFFFF, Type );
|
|
occ = *_pOcc;
|
|
}
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CValueNormalizer::PutValue public
|
|
//
|
|
// Synopsis: Store a 1 byte value without altering it
|
|
//
|
|
// Arguments: [pid] -- property id
|
|
// [byte] -- value
|
|
//
|
|
// History: 25-Oct-93 DwightKr Created.
|
|
//
|
|
// Notes: One byte values are NOT normalized, they are stored as is.
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
void CValueNormalizer::PutValue( PROPID pid, BYTE byte )
|
|
{
|
|
PutValue(pid, (unsigned) byte, VT_UI1);
|
|
}
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CValueNormalizer::PutValue public
|
|
//
|
|
// Synopsis: Store a 1 byte signed value without altering it
|
|
//
|
|
// Arguments: [pid] -- property id
|
|
// [ch] -- value
|
|
//
|
|
// History: 25-Oct-1993 DwightKr Created.
|
|
// 29-Sep-2000 KitmanH Normalize VT_I1 values
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
void CValueNormalizer::PutValue( PROPID pid, CHAR ch )
|
|
{
|
|
PutValue(pid, ( ((BYTE) ch) + 0x80 ) & 0xFF, VT_I1);
|
|
}
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CValueNormalizer::PutValue
|
|
//
|
|
// Synopsis: Store the high byte of an unsigned 2 byte value
|
|
//
|
|
// Arguments: [pid] -- property id
|
|
// [usValue] -- value
|
|
//
|
|
// History: 07-Oct-93 DwightKr Created.
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
void CValueNormalizer::PutValue( PROPID pid, USHORT usValue )
|
|
{
|
|
PutValue(pid, (usValue >> 8) & 0xFF, VT_UI2);
|
|
}
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CValueNormalizer::PutValue public
|
|
//
|
|
// Synopsis: Store the high byte of a signed 2 byte value.
|
|
//
|
|
// Arguments: [pid] -- property id
|
|
// [sValue] -- value
|
|
//
|
|
// Notes: Add the smallest BYTE to this so that we translate numbers
|
|
// into the range above 0. i.e. -32768 maps into 0x00, and 32767
|
|
// maps into 0xFF.
|
|
//
|
|
// History: 07-Oct-93 DwightKr Created.
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
void CValueNormalizer::PutValue( PROPID pid, SHORT sValue )
|
|
{
|
|
PutValue(pid, ((sValue >> 8) + 0x80) & 0xFF, VT_I2);
|
|
}
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CValueNormalizer::PutValue public
|
|
//
|
|
// Synopsis: Store the base-2 log of the ULONG value.
|
|
//
|
|
// Arguments: [pid] -- property id
|
|
// [ulValue] -- value
|
|
//
|
|
// Notes: This convert ULONGs into the range 0 - 31 by taking the Log2
|
|
// of the number.
|
|
//
|
|
// History: 07-Oct-93 DwightKr Created.
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
void CValueNormalizer::PutValue( PROPID pid, ULONG ulValue )
|
|
{
|
|
PutValue(pid, NormULong ( ulValue ), VT_UI4);
|
|
}
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CValueNormalizer::PutValue
|
|
//
|
|
// Synopsis: Store the base-2 log of the signed LONG value.
|
|
//
|
|
// Arguments: [pid] -- property id
|
|
// [lValue] -- value
|
|
//
|
|
// Notes: This converts LONGs into numbers larger than 0. This
|
|
// translates into 64 bins; 32 bins for #'s < 0 & 32 bins for
|
|
// #'s >= 0.
|
|
//
|
|
// History: 07-Oct-93 DwightKr Created.
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
void CValueNormalizer::PutValue( PROPID pid, LONG lValue )
|
|
{
|
|
PutValue(pid, NormLong(lValue), VT_I4);
|
|
}
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CValueNormalizer::PutValue
|
|
//
|
|
// Synopsis: Store the base-10 log of the FLOAT value.
|
|
//
|
|
// Arguments: [pid] -- property id
|
|
// [rValue] -- value
|
|
//
|
|
// Notes: floats fit into a total of 41 bins.
|
|
//
|
|
// History: 07-Oct-93 DwightKr Created.
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
void CValueNormalizer::PutValue( PROPID pid, float rValue )
|
|
{
|
|
PutValue(pid, NormDouble(rValue), VT_R4);
|
|
}
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CValueNormalizer::PutValue
|
|
//
|
|
// Synopsis: Store the base-10 log of the DOUBLE value.
|
|
//
|
|
// Arguments: [pid] -- property id
|
|
// [dValue] -- value
|
|
//
|
|
// Notes: doubles fit into a total of 41 bins.
|
|
//
|
|
// History: 07-Oct-93 DwightKr Created.
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
void CValueNormalizer::PutValue( PROPID pid, double dValue )
|
|
{
|
|
PutValue(pid, NormDouble(dValue), VT_R8);
|
|
}
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CValueNormalizer::PutValue
|
|
//
|
|
// Synopsis: Store the exponent of a large integer
|
|
//
|
|
// Arguments: [pid] -- property id
|
|
// [li] -- value
|
|
//
|
|
// History: 21-Sep-92 BartoszM Created.
|
|
// 04-Feb-93 KyleP Use LARGE_INTEGER
|
|
// 25-Oct-92 DwightKr Copied here & removed extra code &
|
|
// accounted for negative numbers
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
void CValueNormalizer::PutValue( PROPID pid, LARGE_INTEGER liValue )
|
|
{
|
|
unsigned uExponent = NormLargeInteger(liValue);
|
|
|
|
PutValue( pid, uExponent, VT_I8);
|
|
}
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CValueNormalizer::PutValue
|
|
//
|
|
// Synopsis: Store a compressed large integer
|
|
//
|
|
// Arguments: [pid] -- property id
|
|
// [uli] -- value
|
|
//
|
|
// History: 09 Feb 96 AlanW Created.
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
void CValueNormalizer::PutValue( PROPID pid, ULARGE_INTEGER uliValue )
|
|
{
|
|
unsigned uExponent = NormULargeInteger(uliValue);
|
|
|
|
PutValue( pid, uExponent, VT_UI8);
|
|
}
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CValueNormalizer::PutValue
|
|
//
|
|
// Synopsis: Store the least byte of a GUID
|
|
//
|
|
// Arguments: [pid] -- property id
|
|
// [guid] -- value
|
|
//
|
|
// Notes: The GUID generators are guaranteed to modify the TOP DWORD
|
|
// of the 32-byte GUID each time a new GUID is generated.
|
|
// The lower bytes of the GUID is the network address of the
|
|
// card which generated the UUID.
|
|
//
|
|
// We would like to cluster together together objects of a single
|
|
// class (all MS-Word objects together for example). Since it
|
|
// is possible that someone could generate UUIDs for more than
|
|
// one application on a single machine, the lower portion of
|
|
// the UUID will perhaps remain constant between class IDs. The
|
|
// only part of the UUID which is guaranteed to be unique between
|
|
// multiple objects is the field which represents time. It is
|
|
// unlikely that two classes were generated the same second on
|
|
// two different machines.
|
|
//
|
|
// History: 25-Oct-93 DwightKr Created.
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
void CValueNormalizer::PutValue( PROPID pid, GUID const & Guid )
|
|
{
|
|
PutValue(pid, Guid.Data1 & 0xFFFF, VT_CLSID);
|
|
}
|
|
|
|
long CastToLong( double d )
|
|
{
|
|
//
|
|
// bit 63 = sign
|
|
// bits 52 - 62 = exponent
|
|
// bits 0 - 51 = mantissa
|
|
//
|
|
|
|
LARGE_INTEGER * pli = (LARGE_INTEGER *)&d;
|
|
|
|
int exp = (pli->HighPart & 0x7ff00000) >> 20;
|
|
|
|
if ( exp == 0 )
|
|
{
|
|
//
|
|
// Special case: Zero, NaNs, etc.
|
|
//
|
|
|
|
return( 0 );
|
|
}
|
|
|
|
//
|
|
// Subtract off bias
|
|
//
|
|
|
|
exp -= 0x3ff;
|
|
|
|
if ( exp < 0 )
|
|
{
|
|
// Cast of very small number to unsigned long. Loss of precision
|
|
return( 0 );
|
|
}
|
|
else if ( exp > 30 )
|
|
{
|
|
// Cast of very large number to unsigned long. Overflow
|
|
if ( pli->HighPart & 0x80000000 )
|
|
return( LONG_MIN );
|
|
else
|
|
return( LONG_MAX );
|
|
}
|
|
else
|
|
{
|
|
//
|
|
// We need to get the top 32 bits of the mantissa
|
|
// into a dword.
|
|
//
|
|
|
|
unsigned long temp = pli->LowPart >> (32 - 12);
|
|
temp |= pli->HighPart << (32 - 20);
|
|
|
|
//
|
|
// Add the 'hidden' bit of the mantissa. (Since all doubles
|
|
// are normalized to 1.?????? the highest 1 bit isn't stored)
|
|
//
|
|
|
|
temp = temp >> 1;
|
|
temp |= 0x80000000;
|
|
|
|
//
|
|
// Thow away digits to the right of decimal
|
|
//
|
|
|
|
temp = temp >> (31 - exp);
|
|
|
|
//
|
|
// Adjust for sign
|
|
//
|
|
|
|
Win4Assert( (temp & 0x80000000) == 0 );
|
|
long temp2;
|
|
|
|
if ( pli->HighPart & 0x80000000 )
|
|
temp2 = temp * -1;
|
|
else
|
|
temp2 = temp;
|
|
|
|
return( temp2 );
|
|
}
|
|
} //CastToLong
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CValueNormalizer::PutDate
|
|
//
|
|
// Synopsis: Dates are passed in as the number of days (and fractional days)
|
|
// since Jan. 1, 1900. We'll crunch this down to the number of
|
|
// weeks. Dates are passed in a doubles. We'll assume that
|
|
// negative numbers represent dates before Jan. 1, 1900.
|
|
//
|
|
// Arguments: [pid] -- property id
|
|
// [DATE] -- value (double)
|
|
//
|
|
// Notes: Since dates before Jan 1, 1900 are passed as negative numbers
|
|
// we'll need to normalize them to something >= 0.
|
|
//
|
|
// time period resolution # bins
|
|
// =========================== =============== ======
|
|
// year < 10Bil BC -- bin = 0 1
|
|
// 10Bil BC <= year <= 1 BC -- log10 (year) 11
|
|
// 1 BC < year <= 1900 -- year 1902
|
|
// 1901 AD <= year <= 2050 AD -- daily 54787
|
|
// 2051 AD <= year <= 10Bil AD -- log10 (year) 8
|
|
// year > 10Bil AD -- bin = 0xFFFF 1
|
|
//
|
|
//
|
|
// I choose the daily range from 1901 - 2050 since there is a lot
|
|
// of events in the 20th century (WW I, WW II, landing on the
|
|
// moon, my wife's birthday, etc.) that are interesting, and
|
|
// imporant. It is likely that dates outside of this range will
|
|
// be rounded to the nearest year (1492, 1776, 1812, 1867, etc).
|
|
//
|
|
// Also by breaking the log10(year) at 1 BC rather than some other
|
|
// date (such as 0000 AD, or 1 AD) we avoid values in the range
|
|
// 1 BC < year < 1 AD, calculating log10(year) resulting in
|
|
// large negative numbers. Everything in this range should be in
|
|
// bin #12. It also avoids taking log10(0).
|
|
//
|
|
//
|
|
// History: 25-Oct-93 DwightKr Created.
|
|
// 07-Dec-94 KyleP Remove use of floating point
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
|
|
void CValueNormalizer::PutDate( PROPID pid, DATE const & Date )
|
|
{
|
|
const int MinDate = 42; // 2^42 --> ~4.4E12 days --> ~12E9 years --> 12 billion B.C.
|
|
const int MinByYear = 20; // 2^20 --> ~1.0E6 days --> ~2.9E3 years --> 970 B.C.
|
|
const int cMinByYear = (1 << MinByYear) / 365 + 1; // 2873
|
|
const int MaxDaily = (2051 - 1900) * 365; // 55115
|
|
const int MinByYearAD = 15; // 2^15 --> ~32768 days --> ...
|
|
const int MaxDate = 42; // 2^42 --> ~4.4E12 days --> ~12E9 years --> 12 billion A.D.
|
|
|
|
const unsigned FirstBC = 0;
|
|
const unsigned FirstLogBC = FirstBC + 1;
|
|
const unsigned LastLogBC = FirstLogBC + MinDate - MinByYear;
|
|
const unsigned FirstYearBC = LastLogBC + 1;
|
|
const unsigned LastYearBC = FirstYearBC + cMinByYear;
|
|
const unsigned FirstDaily = LastYearBC + 1;
|
|
const unsigned LastDaily = FirstDaily + MaxDaily;
|
|
const unsigned FirstLogAD = LastDaily + 1;
|
|
const unsigned LastLogAD = FirstLogAD + MaxDate - MinByYearAD;
|
|
const unsigned LastAD = 0xFFFF;
|
|
|
|
Win4Assert( LastLogAD < 0xFFFF );
|
|
|
|
unsigned bin;
|
|
BOOL fPositive;
|
|
|
|
int exp = GetExpAndSign( Date, fPositive );
|
|
|
|
if ( !fPositive )
|
|
{
|
|
//
|
|
// Very large negative dates go in first bin
|
|
//
|
|
|
|
if ( exp >= MinDate )
|
|
bin = FirstBC;
|
|
|
|
//
|
|
// Medium size negative dates get 1 bin / power of 2
|
|
//
|
|
|
|
else if ( exp >= MinByYear )
|
|
bin = FirstLogBC - exp + MinByYear;
|
|
|
|
//
|
|
// All other dates before 1900 get 1 bucket per 365 days.
|
|
//
|
|
|
|
else
|
|
{
|
|
long cYears = CastToLong( Date ) / 365;
|
|
|
|
Win4Assert( cYears >= -cMinByYear && cYears <= 0 );
|
|
|
|
bin = FirstYearBC + cYears + cMinByYear;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
//
|
|
// Very large positive dates go in last bin
|
|
//
|
|
|
|
if ( exp >= MaxDate )
|
|
bin = LastAD;
|
|
else
|
|
{
|
|
long cDays = CastToLong( Date );
|
|
|
|
//
|
|
// Dates rather far in the future get 1 bucket / power of 2
|
|
//
|
|
|
|
if ( cDays >= MaxDaily )
|
|
bin = FirstLogAD + exp - MinByYearAD;
|
|
|
|
//
|
|
// Days close to today get 1 bucket per day
|
|
//
|
|
|
|
else
|
|
bin = FirstDaily + cDays;
|
|
}
|
|
}
|
|
|
|
PutValue(pid, bin, VT_DATE);
|
|
} //PutDate
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CValueNormalizer::PutValue
|
|
//
|
|
// Synopsis: Store the hashed value of an 8-byte currency.
|
|
//
|
|
// Arguments: [pid] -- property id
|
|
// [cyValue] -- value
|
|
//
|
|
// Notes: Currency values are stored as a ULONG cents, and a LONG $.
|
|
// We'll ignore the cents portion and store the $ part using
|
|
// the standard LONG storage method.
|
|
//
|
|
// History: 26-Oct-93 DwightKr Created.
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
|
|
void CValueNormalizer::PutValue( PROPID pid, CURRENCY const & cyValue)
|
|
{
|
|
PutValue(pid, NormLong(cyValue.Hi), VT_CY);
|
|
}
|
|
|
|
//+---------------------------------------------------------------------------
|
|
//
|
|
// Member: CValueNormalizer::PutValue
|
|
//
|
|
// Synopsis: Store the number of days since Jan 1, 1980;
|
|
//
|
|
// Arguments: [pid] -- property id
|
|
// [ulValue] -- value
|
|
//
|
|
// History: 07-Oct-93 DwightKr Created.
|
|
//
|
|
// Notes: This algorithym calculates the number of days since Jan 1,
|
|
// 1980; and stores it into a unsigned. FileTimes are divided
|
|
// into the following ranges:
|
|
//
|
|
// FileTime < 1980 => bin 0
|
|
// 1980 <= FileTime <= 1993 week granularity => bins 1 - 729
|
|
// 1994 <= FileTime <= 2160 day granularity => bins 730+
|
|
// FileTime > 2160 => bin 0xFFFF
|
|
//
|
|
//----------------------------------------------------------------------------
|
|
|
|
void CValueNormalizer::PutValue( PROPID pid, FILETIME const & ftValue )
|
|
{
|
|
//
|
|
// Determine the number of days since Jan 1, 1601 by dividing by
|
|
// the number of 100 nanosecond intervals in a day. The result
|
|
// will fit into a ULONG.
|
|
//
|
|
// Then map the result into one of the ranges: before 1980, between
|
|
// 1980 and 1994, between 1994 and 2160, and after 2160. To make
|
|
// the computation easier, we use precomputed values of the number
|
|
// of days from 1601 and the breakpoints of our range.
|
|
//
|
|
|
|
// 100s of nanosecs per day
|
|
const ULONGLONG uliTicsPerDay = 24 * 60 * 60 * (ULONGLONG)10000000;
|
|
|
|
const ULONG ulStart = 138426; // number of days from 1601 to 1980
|
|
const ULONG ulMiddle= 143542; // number of days from 1601 to 1/2/1994
|
|
const ULONG ulEnd = 204535; // number of days from 1601 to 2161
|
|
|
|
ULARGE_INTEGER liValue = {ftValue.dwLowDateTime, ftValue.dwHighDateTime};
|
|
|
|
ULONG ulDays = (ULONG) (liValue.QuadPart / uliTicsPerDay);
|
|
|
|
//
|
|
// We now have the number of days since Jan. 01, 1601 in ulDays.
|
|
// Map into buckets.
|
|
//
|
|
|
|
if (ulDays < ulStart) // Store in bin 0
|
|
{
|
|
PutValue(pid, 0, VT_FILETIME);
|
|
}
|
|
else if (ulDays <= ulMiddle) // Store week granularity
|
|
{
|
|
PutValue(pid, (ulDays + 1 - ulStart) / 7, VT_FILETIME);
|
|
}
|
|
else if (ulDays <= ulEnd) // Store day granularity
|
|
{
|
|
//
|
|
// Bins 0 - 730 are used by the two clauses above. It doesn't
|
|
// really matter if we reuse bin 730 for the start of the next
|
|
// range (this might happen because of the division we do).
|
|
//
|
|
|
|
PutValue(pid, (ulDays + 1 - ulMiddle) + ((ulMiddle - ulStart) / 7),
|
|
VT_FILETIME);
|
|
}
|
|
else // FileTime > 2160
|
|
{
|
|
PutValue(pid, 0xFFFF, VT_FILETIME);
|
|
}
|
|
}
|