Subversion Repositories test_project

/*

** ccio-dma.c:

**      DMA management routines for first generation cache-coherent machines.

**      Program U2/Uturn in "Virtual Mode" and use the I/O MMU.

**

**      (c) Copyright 2000 Grant Grundler

**      (c) Copyright 2000 Ryan Bradetich

**      (c) Copyright 2000 Hewlett-Packard Company

**

** This program is free software; you can redistribute it and/or modify

** it under the terms of the GNU General Public License as published by

** the Free Software Foundation; either version 2 of the License, or

** (at your option) any later version.

**

**

**  "Real Mode" operation refers to U2/Uturn chip operation.

**  U2/Uturn were designed to perform coherency checks w/o using

**  the I/O MMU - basically what x86 does.

**

**  Philipp Rumpf has a "Real Mode" driver for PCX-W machines at:

**      CVSROOT=:pserver:anonymous@198.186.203.37:/cvsroot/linux-parisc

**      cvs -z3 co linux/arch/parisc/kernel/dma-rm.c

**

**  I've rewritten his code to work under TPG's tree. See ccio-rm-dma.c.

**

**  Drawbacks of using Real Mode are:

**      o outbound DMA is slower - U2 won't prefetch data (GSC+ XQL signal).

**      o Inbound DMA less efficient - U2 can't use DMA_FAST attribute.

**      o Ability to do scatter/gather in HW is lost.

**      o Doesn't work under PCX-U/U+ machines since they didn't follow

**        the coherency design originally worked out. Only PCX-W does.

*/

#include <linux/types.h>

#include <linux/kernel.h>

#include <linux/init.h>

#include <linux/mm.h>

#include <linux/spinlock.h>

#include <linux/slab.h>

#include <linux/string.h>

#include <linux/pci.h>

#include <linux/reboot.h>

#include <linux/proc_fs.h>

#include <linux/seq_file.h>

#include <linux/scatterlist.h>

#include <asm/byteorder.h>

#include <asm/cache.h>          /* for L1_CACHE_BYTES */

#include <asm/uaccess.h>

#include <asm/page.h>

#include <asm/dma.h>

#include <asm/io.h>

#include <asm/hardware.h>       /* for register_module() */

#include <asm/parisc-device.h>

/*

** Choose "ccio" since that's what HP-UX calls it.

** Make it easier for folks to migrate from one to the other :^)

*/

#define MODULE_NAME "ccio"

#undef DEBUG_CCIO_RES

#undef DEBUG_CCIO_RUN

#undef DEBUG_CCIO_INIT

#undef DEBUG_CCIO_RUN_SG

#ifdef CONFIG_PROC_FS

/*

 * CCIO_SEARCH_TIME can help measure how fast the bitmap search is.

 * impacts performance though - ditch it if you don't use it.

 */

#define CCIO_SEARCH_TIME

#undef CCIO_MAP_STATS

#else

#undef CCIO_SEARCH_TIME

#undef CCIO_MAP_STATS

#endif

#include <linux/proc_fs.h>

#include <asm/runway.h>         /* for proc_runway_root */

#ifdef DEBUG_CCIO_INIT

#define DBG_INIT(x...)  printk(x)

#else

#define DBG_INIT(x...)

#endif

#ifdef DEBUG_CCIO_RUN

#define DBG_RUN(x...)   printk(x)

#else

#define DBG_RUN(x...)

#endif

#ifdef DEBUG_CCIO_RES

#define DBG_RES(x...)   printk(x)

#else

#define DBG_RES(x...)

#endif

#ifdef DEBUG_CCIO_RUN_SG

#define DBG_RUN_SG(x...) printk(x)

#else

#define DBG_RUN_SG(x...)

#endif

#define CCIO_INLINE     inline

#define WRITE_U32(value, addr) __raw_writel(value, addr)

#define READ_U32(addr) __raw_readl(addr)

#define U2_IOA_RUNWAY 0x580

#define U2_BC_GSC     0x501

#define UTURN_IOA_RUNWAY 0x581

#define UTURN_BC_GSC     0x502

#define IOA_NORMAL_MODE      0x00020080 /* IO_CONTROL to turn on CCIO        */

#define CMD_TLB_DIRECT_WRITE 35         /* IO_COMMAND for I/O TLB Writes     */

#define CMD_TLB_PURGE        33         /* IO_COMMAND to Purge I/O TLB entry */

struct ioa_registers {

        /* Runway Supervisory Set */

        int32_t    unused1[12];

        uint32_t   io_command;             /* Offset 12 */

        uint32_t   io_status;              /* Offset 13 */

        uint32_t   io_control;             /* Offset 14 */

        int32_t    unused2[1];

        /* Runway Auxiliary Register Set */

        uint32_t   io_err_resp;            /* Offset  0 */

        uint32_t   io_err_info;            /* Offset  1 */

        uint32_t   io_err_req;             /* Offset  2 */

        uint32_t   io_err_resp_hi;         /* Offset  3 */

        uint32_t   io_tlb_entry_m;         /* Offset  4 */

        uint32_t   io_tlb_entry_l;         /* Offset  5 */

        uint32_t   unused3[1];

        uint32_t   io_pdir_base;           /* Offset  7 */

        uint32_t   io_io_low_hv;           /* Offset  8 */

        uint32_t   io_io_high_hv;          /* Offset  9 */

        uint32_t   unused4[1];

        uint32_t   io_chain_id_mask;       /* Offset 11 */

        uint32_t   unused5[2];

        uint32_t   io_io_low;              /* Offset 14 */

        uint32_t   io_io_high;             /* Offset 15 */

};

/*

** IOA Registers

** -------------

**

** Runway IO_CONTROL Register (+0x38)

**

** The Runway IO_CONTROL register controls the forwarding of transactions.

**

** | 0  ...  13  |  14 15 | 16 ... 21 | 22 | 23 24 |  25 ... 31 |

** |    HV       |   TLB  |  reserved | HV | mode  |  reserved  |

**

** o mode field indicates the address translation of transactions

**   forwarded from Runway to GSC+:

**       Mode Name     Value        Definition

**       Off (default)   0          Opaque to matching addresses.

**       Include         1          Transparent for matching addresses.

**       Peek            3          Map matching addresses.

**

**       + "Off" mode: Runway transactions which match the I/O range

**         specified by the IO_IO_LOW/IO_IO_HIGH registers will be ignored.

**       + "Include" mode: all addresses within the I/O range specified

**         by the IO_IO_LOW and IO_IO_HIGH registers are transparently

**         forwarded. This is the I/O Adapter's normal operating mode.

**       + "Peek" mode: used during system configuration to initialize the

**         GSC+ bus. Runway Write_Shorts in the address range specified by

**         IO_IO_LOW and IO_IO_HIGH are forwarded through the I/O Adapter

**         *AND* the GSC+ address is remapped to the Broadcast Physical

**         Address space by setting the 14 high order address bits of the

**         32 bit GSC+ address to ones.

**

** o TLB field affects transactions which are forwarded from GSC+ to Runway.

**   "Real" mode is the poweron default.

**

**   TLB Mode  Value  Description

**   Real        0    No TLB translation. Address is directly mapped and the

**                    virtual address is composed of selected physical bits.

**   Error       1    Software fills the TLB manually.

**   Normal      2    IOA fetches IO TLB misses from IO PDIR (in host memory).

**

**

** IO_IO_LOW_HV   +0x60 (HV dependent)

** IO_IO_HIGH_HV  +0x64 (HV dependent)

** IO_IO_LOW      +0x78 (Architected register)

** IO_IO_HIGH     +0x7c (Architected register)

**

** IO_IO_LOW and IO_IO_HIGH set the lower and upper bounds of the

** I/O Adapter address space, respectively.

**

** 0  ... 7 | 8 ... 15 |  16   ...   31 |

** 11111111 | 11111111 |      address   |

**

** Each LOW/HIGH pair describes a disjoint address space region.

** (2 per GSC+ port). Each incoming Runway transaction address is compared

** with both sets of LOW/HIGH registers. If the address is in the range

** greater than or equal to IO_IO_LOW and less than IO_IO_HIGH the transaction

** for forwarded to the respective GSC+ bus.

** Specify IO_IO_LOW equal to or greater than IO_IO_HIGH to avoid specifying

** an address space region.

**

** In order for a Runway address to reside within GSC+ extended address space:

**      Runway Address [0:7]    must identically compare to 8'b11111111

**      Runway Address [8:11]   must be equal to IO_IO_LOW(_HV)[16:19]

**      Runway Address [12:23]  must be greater than or equal to

**                 IO_IO_LOW(_HV)[20:31] and less than IO_IO_HIGH(_HV)[20:31].

**      Runway Address [24:39]  is not used in the comparison.

**

** When the Runway transaction is forwarded to GSC+, the GSC+ address is

** as follows:

**      GSC+ Address[0:3]       4'b1111

**      GSC+ Address[4:29]      Runway Address[12:37]

**      GSC+ Address[30:31]     2'b00

**

** All 4 Low/High registers must be initialized (by PDC) once the lower bus

** is interrogated and address space is defined. The operating system will

** modify the architectural IO_IO_LOW and IO_IO_HIGH registers following

** the PDC initialization.  However, the hardware version dependent IO_IO_LOW

** and IO_IO_HIGH registers should not be subsequently altered by the OS.

**

** Writes to both sets of registers will take effect immediately, bypassing

** the queues, which ensures that subsequent Runway transactions are checked

** against the updated bounds values. However reads are queued, introducing

** the possibility of a read being bypassed by a subsequent write to the same

** register. This sequence can be avoided by having software wait for read

** returns before issuing subsequent writes.

*/

struct ioc {

        struct ioa_registers __iomem *ioc_regs;  /* I/O MMU base address */

        u8  *res_map;                   /* resource map, bit == pdir entry */

        u64 *pdir_base;                 /* physical base address */

        u32 pdir_size;                  /* bytes, function of IOV Space size */

        u32 res_hint;                   /* next available IOVP -

                                           circular search */

        u32 res_size;                   /* size of resource map in bytes */

        spinlock_t res_lock;

#ifdef CCIO_SEARCH_TIME

#define CCIO_SEARCH_SAMPLE 0x100

        unsigned long avg_search[CCIO_SEARCH_SAMPLE];

        unsigned long avg_idx;            /* current index into avg_search */

#endif

#ifdef CCIO_MAP_STATS

        unsigned long used_pages;

        unsigned long msingle_calls;

        unsigned long msingle_pages;

        unsigned long msg_calls;

        unsigned long msg_pages;

        unsigned long usingle_calls;

        unsigned long usingle_pages;

        unsigned long usg_calls;

        unsigned long usg_pages;

#endif

        unsigned short cujo20_bug;

        /* STUFF We don't need in performance path */

        u32 chainid_shift;              /* specify bit location of chain_id */

        struct ioc *next;               /* Linked list of discovered iocs */

        const char *name;               /* device name from firmware */

        unsigned int hw_path;           /* the hardware path this ioc is associatd with */

        struct pci_dev *fake_pci_dev;   /* the fake pci_dev for non-pci devs */

        struct resource mmio_region[2]; /* The "routed" MMIO regions */

};

static struct ioc *ioc_list;

static int ioc_count;

/**************************************************************

*

*   I/O Pdir Resource Management

*

*   Bits set in the resource map are in use.

*   Each bit can represent a number of pages.

*   LSbs represent lower addresses (IOVA's).

*

*   This was was copied from sba_iommu.c. Don't try to unify

*   the two resource managers unless a way to have different

*   allocation policies is also adjusted. We'd like to avoid

*   I/O TLB thrashing by having resource allocation policy

*   match the I/O TLB replacement policy.

*

***************************************************************/

#define IOVP_SIZE PAGE_SIZE

#define IOVP_SHIFT PAGE_SHIFT

#define IOVP_MASK PAGE_MASK

/* Convert from IOVP to IOVA and vice versa. */

#define CCIO_IOVA(iovp,offset) ((iovp) | (offset))

#define CCIO_IOVP(iova) ((iova) & IOVP_MASK)

#define PDIR_INDEX(iovp)    ((iovp)>>IOVP_SHIFT)

#define MKIOVP(pdir_idx)    ((long)(pdir_idx) << IOVP_SHIFT)

#define MKIOVA(iovp,offset) (dma_addr_t)((long)iovp | (long)offset)

/*

** Don't worry about the 150% average search length on a miss.

** If the search wraps around, and passes the res_hint, it will

** cause the kernel to panic anyhow.

*/

#define CCIO_SEARCH_LOOP(ioc, res_idx, mask, size)  \

       for(; res_ptr < res_end; ++res_ptr) { \

               if(0 == (*res_ptr & mask)) { \

                       *res_ptr |= mask; \

                       res_idx = (unsigned int)((unsigned long)res_ptr - (unsigned long)ioc->res_map); \

                       ioc->res_hint = res_idx + (size >> 3); \

                       goto resource_found; \

               } \

       }

#define CCIO_FIND_FREE_MAPPING(ioa, res_idx, mask, size) \

       u##size *res_ptr = (u##size *)&((ioc)->res_map[ioa->res_hint & ~((size >> 3) - 1)]); \

       u##size *res_end = (u##size *)&(ioc)->res_map[ioa->res_size]; \

       CCIO_SEARCH_LOOP(ioc, res_idx, mask, size); \

       res_ptr = (u##size *)&(ioc)->res_map[0]; \

       CCIO_SEARCH_LOOP(ioa, res_idx, mask, size);

/*

** Find available bit in this ioa's resource map.

** Use a "circular" search:

**   o Most IOVA's are "temporary" - avg search time should be small.

** o keep a history of what happened for debugging

** o KISS.

**

** Perf optimizations:

** o search for log2(size) bits at a time.

** o search for available resource bits using byte/word/whatever.

** o use different search for "large" (eg > 4 pages) or "very large"

**   (eg > 16 pages) mappings.

*/

/**

 * ccio_alloc_range - Allocate pages in the ioc's resource map.

 * @ioc: The I/O Controller.

 * @pages_needed: The requested number of pages to be mapped into the

 * I/O Pdir...

 *

 * This function searches the resource map of the ioc to locate a range

 * of available pages for the requested size.

 */

static int

ccio_alloc_range(struct ioc *ioc, size_t size)

{

        unsigned int pages_needed = size >> IOVP_SHIFT;

        unsigned int res_idx;

#ifdef CCIO_SEARCH_TIME

        unsigned long cr_start = mfctl(16);

#endif

        BUG_ON(pages_needed == 0);

        BUG_ON((pages_needed * IOVP_SIZE) > DMA_CHUNK_SIZE);

        DBG_RES("%s() size: %d pages_needed %d\n",

                __FUNCTION__, size, pages_needed);

        /*

        ** "seek and ye shall find"...praying never hurts either...

        ** ggg sacrifices another 710 to the computer gods.

        */

        if (pages_needed <= 8) {

                /*

                 * LAN traffic will not thrash the TLB IFF the same NIC

                 * uses 8 adjacent pages to map seperate payload data.

                 * ie the same byte in the resource bit map.

                 */

#if 0

                /* FIXME: bit search should shift it's way through

                 * an unsigned long - not byte at a time. As it is now,

                 * we effectively allocate this byte to this mapping.

                 */

                unsigned long mask = ~(~0UL >> pages_needed);

                CCIO_FIND_FREE_MAPPING(ioc, res_idx, mask, 8);

#else

                CCIO_FIND_FREE_MAPPING(ioc, res_idx, 0xff, 8);

#endif

        } else if (pages_needed <= 16) {

                CCIO_FIND_FREE_MAPPING(ioc, res_idx, 0xffff, 16);

        } else if (pages_needed <= 32) {

                CCIO_FIND_FREE_MAPPING(ioc, res_idx, ~(unsigned int)0, 32);

#ifdef __LP64__

        } else if (pages_needed <= 64) {

                CCIO_FIND_FREE_MAPPING(ioc, res_idx, ~0UL, 64);

#endif

        } else {

                panic("%s: %s() Too many pages to map. pages_needed: %u\n",

                       __FILE__,  __FUNCTION__, pages_needed);

        }

        panic("%s: %s() I/O MMU is out of mapping resources.\n", __FILE__,

              __FUNCTION__);

resource_found:

        DBG_RES("%s() res_idx %d res_hint: %d\n",

                __FUNCTION__, res_idx, ioc->res_hint);

#ifdef CCIO_SEARCH_TIME

        {

                unsigned long cr_end = mfctl(16);

                unsigned long tmp = cr_end - cr_start;

                /* check for roll over */

                cr_start = (cr_end < cr_start) ?  -(tmp) : (tmp);

        }

        ioc->avg_search[ioc->avg_idx++] = cr_start;

        ioc->avg_idx &= CCIO_SEARCH_SAMPLE - 1;

#endif

#ifdef CCIO_MAP_STATS

        ioc->used_pages += pages_needed;

#endif

        /*

        ** return the bit address.

        */

        return res_idx << 3;

}

#define CCIO_FREE_MAPPINGS(ioc, res_idx, mask, size) \

        u##size *res_ptr = (u##size *)&((ioc)->res_map[res_idx]); \

        BUG_ON((*res_ptr & mask) != mask); \

        *res_ptr &= ~(mask);

/**

 * ccio_free_range - Free pages from the ioc's resource map.

 * @ioc: The I/O Controller.

 * @iova: The I/O Virtual Address.

 * @pages_mapped: The requested number of pages to be freed from the

 * I/O Pdir.

 *

 * This function frees the resouces allocated for the iova.

 */

static void

ccio_free_range(struct ioc *ioc, dma_addr_t iova, unsigned long pages_mapped)

{

        unsigned long iovp = CCIO_IOVP(iova);

        unsigned int res_idx = PDIR_INDEX(iovp) >> 3;

        BUG_ON(pages_mapped == 0);

        BUG_ON((pages_mapped * IOVP_SIZE) > DMA_CHUNK_SIZE);

        BUG_ON(pages_mapped > BITS_PER_LONG);

        DBG_RES("%s():  res_idx: %d pages_mapped %d\n",

                __FUNCTION__, res_idx, pages_mapped);

#ifdef CCIO_MAP_STATS

        ioc->used_pages -= pages_mapped;

#endif

        if(pages_mapped <= 8) {

#if 0

                /* see matching comments in alloc_range */

                unsigned long mask = ~(~0UL >> pages_mapped);

                CCIO_FREE_MAPPINGS(ioc, res_idx, mask, 8);

#else

                CCIO_FREE_MAPPINGS(ioc, res_idx, 0xff, 8);

#endif

        } else if(pages_mapped <= 16) {

                CCIO_FREE_MAPPINGS(ioc, res_idx, 0xffff, 16);

        } else if(pages_mapped <= 32) {

                CCIO_FREE_MAPPINGS(ioc, res_idx, ~(unsigned int)0, 32);

#ifdef __LP64__

        } else if(pages_mapped <= 64) {

                CCIO_FREE_MAPPINGS(ioc, res_idx, ~0UL, 64);

#endif

        } else {

                panic("%s:%s() Too many pages to unmap.\n", __FILE__,

                      __FUNCTION__);

        }

}

/****************************************************************

**

**          CCIO dma_ops support routines

**

*****************************************************************/

typedef unsigned long space_t;

#define KERNEL_SPACE 0

/*

** DMA "Page Type" and Hints

** o if SAFE_DMA isn't set, mapping is for FAST_DMA. SAFE_DMA should be

**   set for subcacheline DMA transfers since we don't want to damage the

**   other part of a cacheline.

** o SAFE_DMA must be set for "memory" allocated via pci_alloc_consistent().

**   This bit tells U2 to do R/M/W for partial cachelines. "Streaming"

**   data can avoid this if the mapping covers full cache lines.

** o STOP_MOST is needed for atomicity across cachelines.

**   Apparently only "some EISA devices" need this.

**   Using CONFIG_ISA is hack. Only the IOA with EISA under it needs

**   to use this hint iff the EISA devices needs this feature.

**   According to the U2 ERS, STOP_MOST enabled pages hurt performance.

** o PREFETCH should *not* be set for cases like Multiple PCI devices

**   behind GSCtoPCI (dino) bus converter. Only one cacheline per GSC

**   device can be fetched and multiply DMA streams will thrash the

**   prefetch buffer and burn memory bandwidth. See 6.7.3 "Prefetch Rules

**   and Invalidation of Prefetch Entries".

**

** FIXME: the default hints need to be per GSC device - not global.

**

** HP-UX dorks: linux device driver programming model is totally different

**    than HP-UX's. HP-UX always sets HINT_PREFETCH since it's drivers

**    do special things to work on non-coherent platforms...linux has to

**    be much more careful with this.

*/

#define IOPDIR_VALID    0x01UL

#define HINT_SAFE_DMA   0x02UL  /* used for pci_alloc_consistent() pages */

#ifdef CONFIG_EISA

#define HINT_STOP_MOST  0x04UL  /* LSL support */

#else

#define HINT_STOP_MOST  0x00UL  /* only needed for "some EISA devices" */

#endif

#define HINT_UDPATE_ENB 0x08UL  /* not used/supported by U2 */

#define HINT_PREFETCH   0x10UL  /* for outbound pages which are not SAFE */

/*

** Use direction (ie PCI_DMA_TODEVICE) to pick hint.

** ccio_alloc_consistent() depends on this to get SAFE_DMA

** when it passes in BIDIRECTIONAL flag.

*/

static u32 hint_lookup[] = {

        [PCI_DMA_BIDIRECTIONAL] = HINT_STOP_MOST | HINT_SAFE_DMA | IOPDIR_VALID,

        [PCI_DMA_TODEVICE]      = HINT_STOP_MOST | HINT_PREFETCH | IOPDIR_VALID,

        [PCI_DMA_FROMDEVICE]    = HINT_STOP_MOST | IOPDIR_VALID,

};

/**

 * ccio_io_pdir_entry - Initialize an I/O Pdir.

 * @pdir_ptr: A pointer into I/O Pdir.

 * @sid: The Space Identifier.

 * @vba: The virtual address.

 * @hints: The DMA Hint.

 *

 * Given a virtual address (vba, arg2) and space id, (sid, arg1),

 * load the I/O PDIR entry pointed to by pdir_ptr (arg0). Each IO Pdir

 * entry consists of 8 bytes as shown below (MSB == bit 0):

 *

 *

 * WORD 0:

 * +------+----------------+-----------------------------------------------+

 * | Phys | Virtual Index  |               Phys                            |

 * | 0:3  |     0:11       |               4:19                            |

 * |4 bits|   12 bits      |              16 bits                          |

 * +------+----------------+-----------------------------------------------+

 * WORD 1:

 * +-----------------------+-----------------------------------------------+

 * |      Phys    |  Rsvd  | Prefetch |Update |Rsvd  |Lock  |Safe  |Valid  |

 * |     20:39    |        | Enable   |Enable |      |Enable|DMA   |       |

 * |    20 bits   | 5 bits | 1 bit    |1 bit  |2 bits|1 bit |1 bit |1 bit  |

 * +-----------------------+-----------------------------------------------+

 *

 * The virtual index field is filled with the results of the LCI

 * (Load Coherence Index) instruction.  The 8 bits used for the virtual

 * index are bits 12:19 of the value returned by LCI.

 */

void CCIO_INLINE

ccio_io_pdir_entry(u64 *pdir_ptr, space_t sid, unsigned long vba,

                   unsigned long hints)

{

        register unsigned long pa;

        register unsigned long ci; /* coherent index */

        /* We currently only support kernel addresses */

        BUG_ON(sid != KERNEL_SPACE);

        mtsp(sid,1);

        /*

        ** WORD 1 - low order word

        ** "hints" parm includes the VALID bit!

        ** "dep" clobbers the physical address offset bits as well.

        */

        pa = virt_to_phys(vba);

        asm volatile("depw  %1,31,12,%0" : "+r" (pa) : "r" (hints));

        ((u32 *)pdir_ptr)[1] = (u32) pa;

        /*

        ** WORD 0 - high order word

        */

#ifdef __LP64__

        /*

        ** get bits 12:15 of physical address

        ** shift bits 16:31 of physical address

        ** and deposit them

        */

        asm volatile ("extrd,u %1,15,4,%0" : "=r" (ci) : "r" (pa));

        asm volatile ("extrd,u %1,31,16,%0" : "+r" (pa) : "r" (pa));

        asm volatile ("depd  %1,35,4,%0" : "+r" (pa) : "r" (ci));

#else

        pa = 0;

#endif

        /*

        ** get CPU coherency index bits

        ** Grab virtual index [0:11]

        ** Deposit virt_idx bits into I/O PDIR word

        */

        asm volatile ("lci %%r0(%%sr1, %1), %0" : "=r" (ci) : "r" (vba));

        asm volatile ("extru %1,19,12,%0" : "+r" (ci) : "r" (ci));

        asm volatile ("depw  %1,15,12,%0" : "+r" (pa) : "r" (ci));

        ((u32 *)pdir_ptr)[0] = (u32) pa;

        /* FIXME: PCX_W platforms don't need FDC/SYNC. (eg C360)

        **        PCX-U/U+ do. (eg C200/C240)

        **        PCX-T'? Don't know. (eg C110 or similar K-class)

        **

        ** See PDC_MODEL/option 0/SW_CAP word for "Non-coherent IO-PDIR bit".

        ** Hopefully we can patch (NOP) these out at boot time somehow.

        **

        ** "Since PCX-U employs an offset hash that is incompatible with

        ** the real mode coherence index generation of U2, the PDIR entry

        ** must be flushed to memory to retain coherence."

        */

        asm volatile("fdc %%r0(%0)" : : "r" (pdir_ptr));

        asm volatile("sync");

}

/**

 * ccio_clear_io_tlb - Remove stale entries from the I/O TLB.

 * @ioc: The I/O Controller.

 * @iovp: The I/O Virtual Page.

 * @byte_cnt: The requested number of bytes to be freed from the I/O Pdir.

 *

 * Purge invalid I/O PDIR entries from the I/O TLB.

 *

 * FIXME: Can we change the byte_cnt to pages_mapped?

 */

static CCIO_INLINE void

ccio_clear_io_tlb(struct ioc *ioc, dma_addr_t iovp, size_t byte_cnt)

{

        u32 chain_size = 1 << ioc->chainid_shift;

        iovp &= IOVP_MASK;      /* clear offset bits, just want pagenum */

        byte_cnt += chain_size;

        while(byte_cnt > chain_size) {

                WRITE_U32(CMD_TLB_PURGE | iovp, &ioc->ioc_regs->io_command);

                iovp += chain_size;

                byte_cnt -= chain_size;

        }

}

/**

 * ccio_mark_invalid - Mark the I/O Pdir entries invalid.

 * @ioc: The I/O Controller.

 * @iova: The I/O Virtual Address.

 * @byte_cnt: The requested number of bytes to be freed from the I/O Pdir.

 *

 * Mark the I/O Pdir entries invalid and blow away the corresponding I/O

 * TLB entries.

 *

 * FIXME: at some threshhold it might be "cheaper" to just blow

 *        away the entire I/O TLB instead of individual entries.

 *

 * FIXME: Uturn has 256 TLB entries. We don't need to purge every

 *        PDIR entry - just once for each possible TLB entry.

 *        (We do need to maker I/O PDIR entries invalid regardless).

 *

 * FIXME: Can we change byte_cnt to pages_mapped?

 */

static CCIO_INLINE void

ccio_mark_invalid(struct ioc *ioc, dma_addr_t iova, size_t byte_cnt)

{

        u32 iovp = (u32)CCIO_IOVP(iova);

        size_t saved_byte_cnt;

        /* round up to nearest page size */

        saved_byte_cnt = byte_cnt = ALIGN(byte_cnt, IOVP_SIZE);

        while(byte_cnt > 0) {

                /* invalidate one page at a time */

                unsigned int idx = PDIR_INDEX(iovp);

                char *pdir_ptr = (char *) &(ioc->pdir_base[idx]);

                BUG_ON(idx >= (ioc->pdir_size / sizeof(u64)));

                pdir_ptr[7] = 0; /* clear only VALID bit */

                /*

                ** FIXME: PCX_W platforms don't need FDC/SYNC. (eg C360)

                **   PCX-U/U+ do. (eg C200/C240)

                ** See PDC_MODEL/option 0/SW_CAP for "Non-coherent IO-PDIR bit".

                **

                ** Hopefully someone figures out how to patch (NOP) the

                ** FDC/SYNC out at boot time.

                */

                asm volatile("fdc %%r0(%0)" : : "r" (pdir_ptr[7]));

                iovp     += IOVP_SIZE;

                byte_cnt -= IOVP_SIZE;

        }

        asm volatile("sync");

        ccio_clear_io_tlb(ioc, CCIO_IOVP(iova), saved_byte_cnt);

}

/****************************************************************

**

**          CCIO dma_ops

**

*****************************************************************/

/**

 * ccio_dma_supported - Verify the IOMMU supports the DMA address range.

 * @dev: The PCI device.

 * @mask: A bit mask describing the DMA address range of the device.

 *

 * This function implements the pci_dma_supported function.

 */

static int

ccio_dma_supported(struct device *dev, u64 mask)

{

        if(dev == NULL) {

                printk(KERN_ERR MODULE_NAME ": EISA/ISA/et al not supported\n");

                BUG();

                return 0;

        }

        /* only support 32-bit devices (ie PCI/GSC) */

        return (int)(mask == 0xffffffffUL);

}

/**

 * ccio_map_single - Map an address range into the IOMMU.

 * @dev: The PCI device.

 * @addr: The start address of the DMA region.

 * @size: The length of the DMA region.

 * @direction: The direction of the DMA transaction (to/from device).

 *

 * This function implements the pci_map_single function.

 */

static dma_addr_t

ccio_map_single(struct device *dev, void *addr, size_t size,

                enum dma_data_direction direction)

{

        int idx;

        struct ioc *ioc;

        unsigned long flags;

        dma_addr_t iovp;

        dma_addr_t offset;

        u64 *pdir_start;

        unsigned long hint = hint_lookup[(int)direction];

        BUG_ON(!dev);

        ioc = GET_IOC(dev);

        BUG_ON(size <= 0);

        /* save offset bits */

        offset = ((unsigned long) addr) & ~IOVP_MASK;

        /* round up to nearest IOVP_SIZE */

        size = ALIGN(size + offset, IOVP_SIZE);

        spin_lock_irqsave(&ioc->res_lock, flags);

#ifdef CCIO_MAP_STATS

        ioc->msingle_calls++;

        ioc->msingle_pages += size >> IOVP_SHIFT;

#endif

        idx = ccio_alloc_range(ioc, size);

        iovp = (dma_addr_t)MKIOVP(idx);

        pdir_start = &(ioc->pdir_base[idx]);

        DBG_RUN("%s() 0x%p -> 0x%lx size: %0x%x\n",

                __FUNCTION__, addr, (long)iovp | offset, size);