/*++

Copyright (c) 1990  Microsoft Corporation

Module Name:

    flush.c

Abstract:

    This module implements MIPS machine dependent kernel functions to flush
    the data and instruction caches and to flush I/O buffers.

Author:

    David N. Cutler (davec) 26-Apr-1990

Environment:

    Kernel mode only.

Revision History:

--*/

#include "ki.h"
ULONG ChangeColor;

//
// Define forward referenced prototyes.
//

VOID
KiChangeColorPageTarget (
    IN PULONG SignalDone,
    IN PVOID NewColor,
    IN PVOID OldColor,
    IN PVOID PageFrame
    );

VOID
KiSweepDcacheTarget (
    IN PULONG SignalDone,
    IN PVOID Parameter1,
    IN PVOID Parameter2,
    IN PVOID Parameter3
    );

VOID
KiSweepIcacheTarget (
    IN PULONG SignalDone,
    IN PVOID Parameter1,
    IN PVOID Parameter2,
    IN PVOID Parameter3
    );

VOID
KiSweepIcacheRangeTarget (
    IN PULONG SignalDone,
    IN PVOID BaseAddress,
    IN PVOID Length,
    IN PVOID Parameter3
    );

VOID
KiFlushIoBuffersTarget (
    IN PULONG SignalDone,
    IN PVOID Mdl,
    IN PVOID ReadOperation,
    IN PVOID DmaOperation
    );

VOID
KeChangeColorPage (
    IN PVOID NewColor,
    IN PVOID OldColor,
    IN ULONG PageFrame
    )

/*++

Routine Description:

    This routine changes the color of a page.

Arguments:

    NewColor - Supplies the page aligned virtual address of the new color
        the page to change.

    OldColor - Supplies the page aligned virtual address of the old color
        of the page to change.

    PageFrame - Supplies the page frame number of the page that is changed.

Return Value:

    None.

--*/

{

    KIRQL OldIrql;
    KAFFINITY TargetProcessors;

    ASSERT(KeGetCurrentIrql() <= KiSynchIrql);

    ChangeColor += 1;

    //
    // Raise IRQL to synchronization level to prevent a context switch.
    //

#if !defined(NT_UP)

    OldIrql = KeRaiseIrqlToSynchLevel();

    //
    // Compute the set of target processors and send the change color
    // parameters to the target processors, if any, for execution.
    //

    TargetProcessors = KeActiveProcessors & PCR->NotMember;
    if (TargetProcessors != 0) {
        KiIpiSendPacket(TargetProcessors,
                        KiChangeColorPageTarget,
                        (PVOID)NewColor,
                        (PVOID)OldColor,
                        (PVOID)PageFrame);
    }

#endif

    //
    // Change the color of the page on the current processor.
    //

    HalChangeColorPage(NewColor, OldColor, PageFrame);

    //
    // Wait until all target processors have finished changing the color
    // of the page.
    //

#if !defined(NT_UP)

    if (TargetProcessors != 0) {
        KiIpiStallOnPacketTargets();
    }

    //
    // Lower IRQL to its previous level and return.
    //

    KeLowerIrql(OldIrql);

#endif

    return;
}

VOID
KiChangeColorPageTarget (
    IN PULONG SignalDone,
    IN PVOID NewColor,
    IN PVOID OldColor,
    IN PVOID PageFrame
    )

/*++

Routine Description:

    This is the target function for changing the color of a page.

Arguments:

    SignalDone Supplies a pointer to a variable that is cleared when the
        requested operation has been performed.

    NewColor - Supplies the page aligned virtual address of the new color
        the page to change.

    OldColor - Supplies the page aligned virtual address of the old color
        of the page to change.

    PageFrame - Supplies the page frame number of the page that is changed.

Return Value:

    None.

--*/

{


    //
    // Change the color of the page on the current processor and clear
    // change color packet address to signal the source to continue.
    //

#if !defined(NT_UP)

    HalChangeColorPage(NewColor, OldColor, (ULONG)PageFrame);
    KiIpiSignalPacketDone(SignalDone);

#endif

    return;
}

VOID
KeSweepDcache (
    IN BOOLEAN AllProcessors
    )

/*++

Routine Description:

    This function flushes the data cache on all processors that are currently
    running threads which are children of the current process or flushes the
    data cache on all processors in the host configuration.

Arguments:

    AllProcessors - Supplies a boolean value that determines which data
        caches are flushed.

Return Value:

    None.

--*/

{

    KIRQL OldIrql;
    KAFFINITY TargetProcessors;

    ASSERT(KeGetCurrentIrql() <= KiSynchIrql);

    //
    // Raise IRQL to synchronization level to prevent a context switch.
    //

#if !defined(NT_UP)

    OldIrql = KeRaiseIrqlToSynchLevel();

    //
    // Compute the set of target processors and send the sweep parameters
    // to the target processors, if any, for execution.
    //

    TargetProcessors = KeActiveProcessors & PCR->NotMember;
    if (TargetProcessors != 0) {
        KiIpiSendPacket(TargetProcessors,
                        KiSweepDcacheTarget,
                        NULL,
                        NULL,
                        NULL);
    }

#endif

    //
    // Sweep the data cache on the current processor.
    //

    HalSweepDcache();

    //
    // Wait until all target processors have finished sweeping the their
    // data cache.
    //

#if !defined(NT_UP)

    if (TargetProcessors != 0) {
        KiIpiStallOnPacketTargets();
    }

    //
    // Lower IRQL to its previous level and return.
    //

    KeLowerIrql(OldIrql);

#endif

    return;
}

VOID
KiSweepDcacheTarget (
    IN PULONG SignalDone,
    IN PVOID Parameter1,
    IN PVOID Parameter2,
    IN PVOID Parameter3
    )

/*++

Routine Description:

    This is the target function for sweeping the data cache on target
    processors.

Arguments:

    SignalDone Supplies a pointer to a variable that is cleared when the
        requested operation has been performed.

    Parameter1 - Parameter3 - Not used.

Return Value:

    None.

--*/

{

    //
    // Sweep the data cache on the current processor and clear the sweep
    // data cache packet address to signal the source to continue.
    //

#if !defined(NT_UP)

    HalSweepDcache();
    KiIpiSignalPacketDone(SignalDone);

#endif

    return;
}

VOID
KeSweepIcache (
    IN BOOLEAN AllProcessors
    )

/*++

Routine Description:

    This function flushes the instruction cache on all processors that are
    currently running threads which are children of the current process or
    flushes the instruction cache on all processors in the host configuration.

Arguments:

    AllProcessors - Supplies a boolean value that determines which instruction
        caches are flushed.

Return Value:

    None.

--*/

{

    KIRQL OldIrql;
    KAFFINITY TargetProcessors;

    ASSERT(KeGetCurrentIrql() <= KiSynchIrql);

    //
    // Raise IRQL to synchrnization level to prevent a context switch.
    //

#if !defined(NT_UP)

    OldIrql = KeRaiseIrqlToSynchLevel();

    //
    // Compute the set of target processors and send the sweep parameters
    // to the target processors, if any, for execution.
    //

    TargetProcessors = KeActiveProcessors & PCR->NotMember;
    if (TargetProcessors != 0) {
        KiIpiSendPacket(TargetProcessors,
                        KiSweepIcacheTarget,
                        NULL,
                        NULL,
                        NULL);
    }

#endif

    //
    // Sweep the instruction cache on the current processor.
    //

    HalSweepIcache();
    HalSweepDcache();

    //
    // Wait until all target processors have finished sweeping the their
    // instruction cache.
    //

#if !defined(NT_UP)

    if (TargetProcessors != 0) {
        KiIpiStallOnPacketTargets();
    }

    //
    // Lower IRQL to its previous level and return.
    //

    KeLowerIrql(OldIrql);

#endif

    return;
}

VOID
KiSweepIcacheTarget (
    IN PULONG SignalDone,
    IN PVOID Parameter1,
    IN PVOID Parameter2,
    IN PVOID Parameter3
    )

/*++

Routine Description:

    This is the target function for sweeping the instruction cache on
    target processors.

Arguments:

    SignalDone Supplies a pointer to a variable that is cleared when the
        requested operation has been performed.

    Parameter1 - Parameter3 - Not used.

Return Value:

    None.

--*/

{

    //
    // Sweep the instruction cache on the current processor and clear
    // the sweep instruction cache packet address to signal the source
    // to continue.
    //

#if !defined(NT_UP)

    HalSweepIcache();
    HalSweepDcache();
    KiIpiSignalPacketDone(SignalDone);

#endif

    return;
}

VOID
KeSweepIcacheRange (
    IN BOOLEAN AllProcessors,
    IN PVOID BaseAddress,
    IN ULONG Length
    )

/*++

Routine Description:

    This function flushes the an range of virtual addresses from the primary
    instruction cache on all processors that are currently running threads
    which are children of the current process or flushes the range of virtual
    addresses from the primary instruction cache on all processors in the host
    configuration.

Arguments:

    AllProcessors - Supplies a boolean value that determines which instruction
        caches are flushed.

    BaseAddress - Supplies a pointer to the base of the range that is flushed.

    Length - Supplies the length of the range that is flushed if the base
        address is specified.

Return Value:

    None.

--*/

{

    ULONG Offset;
    KIRQL OldIrql;
    KAFFINITY TargetProcessors;

    ASSERT(KeGetCurrentIrql() <= KiSynchIrql);

    //
    // If the length of the range is greater than the size of the primary
    // instruction cache, then set the length of the flush to the size of
    // the primary instruction cache and set the base address of zero.
    //
    // N.B. It is assumed that the size of the primary instruction and
    //      data caches are the same.
    //

    if (Length > PCR->FirstLevelIcacheSize) {
        BaseAddress = (PVOID)0;
        Length = PCR->FirstLevelIcacheSize;
    }

    //
    // Raise IRQL to synchronization level to prevent a context switch.
    //

#if !defined(NT_UP)

    OldIrql = KeRaiseIrqlToSynchLevel();

    //
    // Compute the set of target processors, and send the sweep range
    // parameters to the target processors, if any, for execution.
    //

    TargetProcessors = KeActiveProcessors & PCR->NotMember;
    if (TargetProcessors != 0) {
        KiIpiSendPacket(TargetProcessors,
                        KiSweepIcacheRangeTarget,
                        (PVOID)BaseAddress,
                        (PVOID)Length,
                        NULL);
    }

#endif

    //
    // Flush the specified range of virtual addresses from the primary
    // instruction cache.
    //

    Offset = (ULONG)BaseAddress & PCR->IcacheAlignment;
    HalSweepIcacheRange((PVOID)((ULONG)BaseAddress & ~PCR->IcacheAlignment),
                        (Offset + Length + PCR->IcacheAlignment) & ~PCR->IcacheAlignment);

    Offset = (ULONG)BaseAddress & PCR->DcacheAlignment;
    HalSweepDcacheRange((PVOID)((ULONG)BaseAddress & ~PCR->DcacheAlignment),
                        (Offset + Length + PCR->DcacheAlignment) & ~PCR->DcacheAlignment);

    //
    // Wait until all target processors have finished sweeping the specified
    // range of addresses from the instruction cache.
    //

#if !defined(NT_UP)

    if (TargetProcessors != 0) {
        KiIpiStallOnPacketTargets();
    }

    //
    // Lower IRQL to its previous level and return.
    //

    KeLowerIrql(OldIrql);

#endif

    return;
}

VOID
KiSweepIcacheRangeTarget (
    IN PULONG SignalDone,
    IN PVOID BaseAddress,
    IN PVOID Length,
    IN PVOID Parameter3
    )

/*++

Routine Description:

    This is the target function for sweeping a range of addresses from the
    instruction cache.
    processors.

Arguments:

    SignalDone Supplies a pointer to a variable that is cleared when the
        requested operation has been performed.

    BaseAddress - Supplies a pointer to the base of the range that is flushed.

    Length - Supplies the length of the range that is flushed if the base
        address is specified.

    Parameter3 - Not used.

Return Value:

    None.

--*/

{

    ULONG Offset;

    //
    // Sweep the specified instruction cache range on the current processor.
    //

#if !defined(NT_UP)

    Offset = (ULONG)(BaseAddress) & PCR->IcacheAlignment;
    HalSweepIcacheRange((PVOID)((ULONG)(BaseAddress) & ~PCR->IcacheAlignment),
                        (Offset + (ULONG)Length + PCR->IcacheAlignment) & ~PCR->IcacheAlignment);

    Offset = (ULONG)(BaseAddress) & PCR->DcacheAlignment;
    HalSweepDcacheRange((PVOID)((ULONG)(BaseAddress) & ~PCR->DcacheAlignment),
                        (Offset + (ULONG)Length + PCR->DcacheAlignment) & ~PCR->DcacheAlignment);

    KiIpiSignalPacketDone(SignalDone);

#endif

    return;
}

VOID
KeFlushIoBuffers (
    IN PMDL Mdl,
    IN BOOLEAN ReadOperation,
    IN BOOLEAN DmaOperation
    )

/*++

Routine Description:

    This function flushes the I/O buffer specified by the memory descriptor
    list from the data cache on all processors.

Arguments:

    Mdl - Supplies a pointer to a memory descriptor list that describes the
        I/O buffer location.

    ReadOperation - Supplies a boolean value that determines whether the I/O
        operation is a read into memory.

    DmaOperation - Supplies a boolean value that determines whether the I/O
        operation is a DMA operation.

Return Value:

    None.

--*/

{

    KIRQL OldIrql;
    KAFFINITY TargetProcessors;

    ASSERT(KeGetCurrentIrql() <= KiSynchIrql);

    //
    // If the operation is a DMA operation, then check if the flush
    // can be avoided because the host system supports the right set
    // of cache coherency attributes. Otherwise, the flush can also
    // be avoided if the operation is a programmed I/O and not a page
    // read.
    //

    if (DmaOperation != FALSE) {
        if (ReadOperation != FALSE) {
            if ((KiDmaIoCoherency & DMA_READ_ICACHE_INVALIDATE) != 0) {

                ASSERT((KiDmaIoCoherency & DMA_READ_DCACHE_INVALIDATE) != 0);

                return;

            } else if (((Mdl->MdlFlags & MDL_IO_PAGE_READ) == 0) &&
                ((KiDmaIoCoherency & DMA_READ_DCACHE_INVALIDATE) != 0)) {
                return;
            }

        } else if ((KiDmaIoCoherency & DMA_WRITE_DCACHE_SNOOP) != 0) {
            return;
        }

    } else if ((Mdl->MdlFlags & MDL_IO_PAGE_READ) == 0) {
        return;
    }

    //
    // Either the operation is a DMA operation and the right coherency
    // atributes are not supported by the host system, or the operation
    // is programmed I/O and a page read.
    //
    // Raise IRQL to synchronization level to prevent a context switch.
    //

    OldIrql = KeRaiseIrqlToSynchLevel();

    //
    // Compute the set of target processors, and send the flush I/O
    // parameters to the target processors, if any, for execution.
    //

#if !defined(NT_UP)

    TargetProcessors = KeActiveProcessors & PCR->NotMember;
    if (TargetProcessors != 0) {
        KiIpiSendPacket(TargetProcessors,
                        KiFlushIoBuffersTarget,
                        (PVOID)Mdl,
                        (PVOID)((ULONG)ReadOperation),
                        (PVOID)((ULONG)DmaOperation));
    }

#endif

    //
    // Flush I/O buffer on current processor.
    //

    HalFlushIoBuffers(Mdl, ReadOperation, DmaOperation);

    //
    // Wait until all target processors have finished flushing the
    // specified I/O buffer.
    //

#if !defined(NT_UP)

    if (TargetProcessors != 0) {
        KiIpiStallOnPacketTargets();
    }

#endif

    //
    // Lower IRQL to its previous level and return.
    //

    KeLowerIrql(OldIrql);
    return;
}

VOID
KiFlushIoBuffersTarget (
    IN PULONG SignalDone,
    IN PVOID Mdl,
    IN PVOID ReadOperation,
    IN PVOID DmaOperation
    )

/*++

Routine Description:

    This is the target function for flushing an I/O buffer on target
    processors.

Arguments:

    SignalDone Supplies a pointer to a variable that is cleared when the
        requested operation has been performed.

    Mdl - Supplies a pointer to a memory descriptor list that describes the
        I/O buffer location.

    ReadOperation - Supplies a boolean value that determines whether the I/O
        operation is a read into memory.

    DmaOperation - Supplies a boolean value that determines whether the I/O
        operation is a DMA operation.

Return Value:

    None.

--*/

{

    //
    // Flush the specified I/O buffer on the current processor.
    //

#if !defined(NT_UP)

    HalFlushIoBuffers((PMDL)Mdl,
                      (BOOLEAN)((ULONG)ReadOperation),
                      (BOOLEAN)((ULONG)DmaOperation));

    KiIpiSignalPacketDone(SignalDone);

#endif

    return;
}