Subversion Repositories or1k

/* $Id: sbus.c,v 1.1.1.1 2004-04-15 01:34:38 phoenix Exp $

 * sbus.c: UltraSparc SBUS controller support.

 *

 * Copyright (C) 1999 David S. Miller (davem@redhat.com)

 */

#include <linux/kernel.h>

#include <linux/types.h>

#include <linux/mm.h>

#include <linux/spinlock.h>

#include <linux/slab.h>

#include <linux/init.h>

#include <asm/page.h>

#include <asm/sbus.h>

#include <asm/io.h>

#include <asm/upa.h>

#include <asm/cache.h>

#include <asm/dma.h>

#include <asm/irq.h>

#include <asm/starfire.h>

#include "iommu_common.h"

/* These should be allocated on an SMP_CACHE_BYTES

 * aligned boundry for optimal performance.

 *

 * On SYSIO, using an 8K page size we have 1GB of SBUS

 * DMA space mapped.  We divide this space into equally

 * sized clusters.  Currently we allow clusters up to a

 * size of 1MB.  If anything begins to generate DMA

 * mapping requests larger than this we will need to

 * increase things a bit.

 */

#define NCLUSTERS       8UL

#define ONE_GIG         (1UL * 1024UL * 1024UL * 1024UL)

#define CLUSTER_SIZE    (ONE_GIG / NCLUSTERS)

#define CLUSTER_MASK    (CLUSTER_SIZE - 1)

#define CLUSTER_NPAGES  (CLUSTER_SIZE >> IO_PAGE_SHIFT)

#define MAP_BASE        ((u32)0xc0000000)

struct sbus_iommu {

/*0x00*/spinlock_t              lock;

/*0x08*/iopte_t                 *page_table;

/*0x10*/unsigned long           strbuf_regs;

/*0x18*/unsigned long           iommu_regs;

/*0x20*/unsigned long           sbus_control_reg;

/*0x28*/volatile unsigned long  strbuf_flushflag;

        /* If NCLUSTERS is ever decresed to 4 or lower,

         * you must increase the size of the type of

         * these counters.  You have been duly warned. -DaveM

         */

/*0x30*/struct {

                u16     next;

                u16     flush;

        } alloc_info[NCLUSTERS];

        /* The lowest used consistent mapping entry.  Since

         * we allocate consistent maps out of cluster 0 this

         * is relative to the beginning of closter 0.

         */

/*0x50*/u32             lowest_consistent_map;

};

/* Offsets from iommu_regs */

#define SYSIO_IOMMUREG_BASE     0x2400UL

#define IOMMU_CONTROL   (0x2400UL - 0x2400UL)   /* IOMMU control register */

#define IOMMU_TSBBASE   (0x2408UL - 0x2400UL)   /* TSB base address register */

#define IOMMU_FLUSH     (0x2410UL - 0x2400UL)   /* IOMMU flush register */

#define IOMMU_VADIAG    (0x4400UL - 0x2400UL)   /* SBUS virtual address diagnostic */

#define IOMMU_TAGCMP    (0x4408UL - 0x2400UL)   /* TLB tag compare diagnostics */

#define IOMMU_LRUDIAG   (0x4500UL - 0x2400UL)   /* IOMMU LRU queue diagnostics */

#define IOMMU_TAGDIAG   (0x4580UL - 0x2400UL)   /* TLB tag diagnostics */

#define IOMMU_DRAMDIAG  (0x4600UL - 0x2400UL)   /* TLB data RAM diagnostics */

#define IOMMU_DRAM_VALID        (1UL << 30UL)

static void __iommu_flushall(struct sbus_iommu *iommu)

{

        unsigned long tag = iommu->iommu_regs + IOMMU_TAGDIAG;

        int entry;

        for (entry = 0; entry < 16; entry++) {

                upa_writeq(0, tag);

                tag += 8UL;

        }

        upa_readq(iommu->sbus_control_reg);

        for (entry = 0; entry < NCLUSTERS; entry++) {

                iommu->alloc_info[entry].flush =

                        iommu->alloc_info[entry].next;

        }

}

static void iommu_flush(struct sbus_iommu *iommu, u32 base, unsigned long npages)

{

        while (npages--)

                upa_writeq(base + (npages << IO_PAGE_SHIFT),

                           iommu->iommu_regs + IOMMU_FLUSH);

        upa_readq(iommu->sbus_control_reg);

}

/* Offsets from strbuf_regs */

#define SYSIO_STRBUFREG_BASE    0x2800UL

#define STRBUF_CONTROL  (0x2800UL - 0x2800UL)   /* Control */

#define STRBUF_PFLUSH   (0x2808UL - 0x2800UL)   /* Page flush/invalidate */

#define STRBUF_FSYNC    (0x2810UL - 0x2800UL)   /* Flush synchronization */

#define STRBUF_DRAMDIAG (0x5000UL - 0x2800UL)   /* data RAM diagnostic */

#define STRBUF_ERRDIAG  (0x5400UL - 0x2800UL)   /* error status diagnostics */

#define STRBUF_PTAGDIAG (0x5800UL - 0x2800UL)   /* Page tag diagnostics */

#define STRBUF_LTAGDIAG (0x5900UL - 0x2800UL)   /* Line tag diagnostics */

#define STRBUF_TAG_VALID        0x02UL

static void strbuf_flush(struct sbus_iommu *iommu, u32 base, unsigned long npages)

{

        iommu->strbuf_flushflag = 0UL;

        while (npages--)

                upa_writeq(base + (npages << IO_PAGE_SHIFT),

                           iommu->strbuf_regs + STRBUF_PFLUSH);

        /* Whoopee cushion! */

        upa_writeq(__pa(&iommu->strbuf_flushflag),

                   iommu->strbuf_regs + STRBUF_FSYNC);

        upa_readq(iommu->sbus_control_reg);

        while (iommu->strbuf_flushflag == 0UL)

                membar("#LoadLoad");

}

static iopte_t *alloc_streaming_cluster(struct sbus_iommu *iommu, unsigned long npages)

{

        iopte_t *iopte, *limit, *first;

        unsigned long cnum, ent, flush_point;

        cnum = 0;

        while ((1UL << cnum) < npages)

                cnum++;

        iopte  = iommu->page_table + (cnum * CLUSTER_NPAGES);

        if (cnum == 0)

                limit = (iommu->page_table +

                         iommu->lowest_consistent_map);

        else

                limit = (iopte + CLUSTER_NPAGES);

        iopte += ((ent = iommu->alloc_info[cnum].next) << cnum);

        flush_point = iommu->alloc_info[cnum].flush;

        first = iopte;

        for (;;) {

                if (iopte_val(*iopte) == 0UL) {

                        if ((iopte + (1 << cnum)) >= limit)

                                ent = 0;

                        else

                                ent = ent + 1;

                        iommu->alloc_info[cnum].next = ent;

                        if (ent == flush_point)

                                __iommu_flushall(iommu);

                        break;

                }

                iopte += (1 << cnum);

                ent++;

                if (iopte >= limit) {

                        iopte = (iommu->page_table + (cnum * CLUSTER_NPAGES));

                        ent = 0;

                }

                if (ent == flush_point)

                        __iommu_flushall(iommu);

                if (iopte == first)

                        goto bad;

        }

        /* I've got your streaming cluster right here buddy boy... */

        return iopte;

bad:

        printk(KERN_EMERG "sbus: alloc_streaming_cluster of npages(%ld) failed!\n",

               npages);

        return NULL;

}

static void free_streaming_cluster(struct sbus_iommu *iommu, u32 base, unsigned long npages)

{

        unsigned long cnum, ent;

        iopte_t *iopte;

        cnum = 0;

        while ((1UL << cnum) < npages)

                cnum++;

        ent = (base & CLUSTER_MASK) >> (IO_PAGE_SHIFT + cnum);

        iopte = iommu->page_table + ((base - MAP_BASE) >> IO_PAGE_SHIFT);

        iopte_val(*iopte) = 0UL;

        /* If the global flush might not have caught this entry,

         * adjust the flush point such that we will flush before

         * ever trying to reuse it.

         */

#define between(X,Y,Z)  (((Z) - (Y)) >= ((X) - (Y)))

        if (between(ent, iommu->alloc_info[cnum].next, iommu->alloc_info[cnum].flush))

                iommu->alloc_info[cnum].flush = ent;

#undef between

}

/* We allocate consistent mappings from the end of cluster zero. */

static iopte_t *alloc_consistent_cluster(struct sbus_iommu *iommu, unsigned long npages)

{

        iopte_t *iopte;

        iopte = iommu->page_table + (1 * CLUSTER_NPAGES);

        while (iopte > iommu->page_table) {

                iopte--;

                if (!(iopte_val(*iopte) & IOPTE_VALID)) {

                        unsigned long tmp = npages;

                        while (--tmp) {

                                iopte--;

                                if (iopte_val(*iopte) & IOPTE_VALID)

                                        break;

                        }

                        if (tmp == 0) {

                                u32 entry = (iopte - iommu->page_table);

                                if (entry < iommu->lowest_consistent_map)

                                        iommu->lowest_consistent_map = entry;

                                return iopte;

                        }

                }

        }

        return NULL;

}

static void free_consistent_cluster(struct sbus_iommu *iommu, u32 base, unsigned long npages)

{

        iopte_t *iopte = iommu->page_table + ((base - MAP_BASE) >> IO_PAGE_SHIFT);

        if ((iopte - iommu->page_table) == iommu->lowest_consistent_map) {

                iopte_t *walk = iopte + npages;

                iopte_t *limit;

                limit = iommu->page_table + CLUSTER_NPAGES;

                while (walk < limit) {

                        if (iopte_val(*walk) != 0UL)

                                break;

                        walk++;

                }

                iommu->lowest_consistent_map =

                        (walk - iommu->page_table);

        }

        while (npages--)

                *iopte++ = __iopte(0UL);

}

void *sbus_alloc_consistent(struct sbus_dev *sdev, size_t size, dma_addr_t *dvma_addr)

{

        unsigned long order, first_page, flags;

        struct sbus_iommu *iommu;

        iopte_t *iopte;

        void *ret;

        int npages;

        if (size <= 0 || sdev == NULL || dvma_addr == NULL)

                return NULL;

        size = IO_PAGE_ALIGN(size);

        order = get_order(size);

        if (order >= 10)

                return NULL;

        first_page = __get_free_pages(GFP_KERNEL, order);

        if (first_page == 0UL)

                return NULL;

        memset((char *)first_page, 0, PAGE_SIZE << order);

        iommu = sdev->bus->iommu;

        spin_lock_irqsave(&iommu->lock, flags);

        iopte = alloc_consistent_cluster(iommu, size >> IO_PAGE_SHIFT);

        if (iopte == NULL) {

                spin_unlock_irqrestore(&iommu->lock, flags);

                free_pages(first_page, order);

                return NULL;

        }

        /* Ok, we're committed at this point. */

        *dvma_addr = MAP_BASE + ((iopte - iommu->page_table) << IO_PAGE_SHIFT);

        ret = (void *) first_page;

        npages = size >> IO_PAGE_SHIFT;

        while (npages--) {

                *iopte++ = __iopte(IOPTE_VALID | IOPTE_CACHE | IOPTE_WRITE |

                                   (__pa(first_page) & IOPTE_PAGE));

                first_page += IO_PAGE_SIZE;

        }

        iommu_flush(iommu, *dvma_addr, size >> IO_PAGE_SHIFT);

        spin_unlock_irqrestore(&iommu->lock, flags);

        return ret;

}

void sbus_free_consistent(struct sbus_dev *sdev, size_t size, void *cpu, dma_addr_t dvma)

{

        unsigned long order, npages;

        struct sbus_iommu *iommu;

        if (size <= 0 || sdev == NULL || cpu == NULL)

                return;

        npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT;

        iommu = sdev->bus->iommu;

        spin_lock_irq(&iommu->lock);

        free_consistent_cluster(iommu, dvma, npages);

        iommu_flush(iommu, dvma, npages);

        spin_unlock_irq(&iommu->lock);

        order = get_order(size);

        if (order < 10)

                free_pages((unsigned long)cpu, order);

}

dma_addr_t sbus_map_single(struct sbus_dev *sdev, void *ptr, size_t size, int dir)

{

        struct sbus_iommu *iommu = sdev->bus->iommu;

        unsigned long npages, pbase, flags;

        iopte_t *iopte;

        u32 dma_base, offset;

        unsigned long iopte_bits;

        if (dir == SBUS_DMA_NONE)

                BUG();

        pbase = (unsigned long) ptr;

        offset = (u32) (pbase & ~IO_PAGE_MASK);

        size = (IO_PAGE_ALIGN(pbase + size) - (pbase & IO_PAGE_MASK));

        pbase = (unsigned long) __pa(pbase & IO_PAGE_MASK);

        spin_lock_irqsave(&iommu->lock, flags);

        npages = size >> IO_PAGE_SHIFT;

        iopte = alloc_streaming_cluster(iommu, npages);

        if (iopte == NULL)

                goto bad;

        dma_base = MAP_BASE + ((iopte - iommu->page_table) << IO_PAGE_SHIFT);

        npages = size >> IO_PAGE_SHIFT;

        iopte_bits = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE;

        if (dir != SBUS_DMA_TODEVICE)

                iopte_bits |= IOPTE_WRITE;

        while (npages--) {

                *iopte++ = __iopte(iopte_bits | (pbase & IOPTE_PAGE));

                pbase += IO_PAGE_SIZE;

        }

        npages = size >> IO_PAGE_SHIFT;

        spin_unlock_irqrestore(&iommu->lock, flags);

        return (dma_base | offset);

bad:

        spin_unlock_irqrestore(&iommu->lock, flags);

        BUG();

        return 0;

}

void sbus_unmap_single(struct sbus_dev *sdev, dma_addr_t dma_addr, size_t size, int direction)

{

        struct sbus_iommu *iommu = sdev->bus->iommu;

        u32 dma_base = dma_addr & IO_PAGE_MASK;

        unsigned long flags;

        size = (IO_PAGE_ALIGN(dma_addr + size) - dma_base);

        spin_lock_irqsave(&iommu->lock, flags);

        free_streaming_cluster(iommu, dma_base, size >> IO_PAGE_SHIFT);

        strbuf_flush(iommu, dma_base, size >> IO_PAGE_SHIFT);

        spin_unlock_irqrestore(&iommu->lock, flags);

}

#define SG_ENT_PHYS_ADDRESS(SG) \

        ((SG)->address ? \

         __pa((SG)->address) : \

         (__pa(page_address((SG)->page)) + (SG)->offset))

static inline void fill_sg(iopte_t *iopte, struct scatterlist *sg, int nused, int nelems, unsigned long iopte_bits)

{

        struct scatterlist *dma_sg = sg;

        struct scatterlist *sg_end = sg + nelems;

        int i;

        for (i = 0; i < nused; i++) {

                unsigned long pteval = ~0UL;

                u32 dma_npages;

                dma_npages = ((dma_sg->dma_address & (IO_PAGE_SIZE - 1UL)) +

                              dma_sg->dma_length +

                              ((IO_PAGE_SIZE - 1UL))) >> IO_PAGE_SHIFT;

                do {

                        unsigned long offset;

                        signed int len;

                        /* If we are here, we know we have at least one

                         * more page to map.  So walk forward until we

                         * hit a page crossing, and begin creating new

                         * mappings from that spot.

                         */

                        for (;;) {

                                unsigned long tmp;

                                tmp = (unsigned long) SG_ENT_PHYS_ADDRESS(sg);

                                len = sg->length;

                                if (((tmp ^ pteval) >> IO_PAGE_SHIFT) != 0UL) {

                                        pteval = tmp & IO_PAGE_MASK;

                                        offset = tmp & (IO_PAGE_SIZE - 1UL);

                                        break;

                                }

                                if (((tmp ^ (tmp + len - 1UL)) >> IO_PAGE_SHIFT) != 0UL) {

                                        pteval = (tmp + IO_PAGE_SIZE) & IO_PAGE_MASK;

                                        offset = 0UL;

                                        len -= (IO_PAGE_SIZE - (tmp & (IO_PAGE_SIZE - 1UL)));

                                        break;

                                }

                                sg++;

                        }

                        pteval = ((pteval & IOPTE_PAGE) | iopte_bits);

                        while (len > 0) {

                                *iopte++ = __iopte(pteval);

                                pteval += IO_PAGE_SIZE;

                                len -= (IO_PAGE_SIZE - offset);

                                offset = 0;

                                dma_npages--;

                        }

                        pteval = (pteval & IOPTE_PAGE) + len;

                        sg++;

                        /* Skip over any tail mappings we've fully mapped,

                         * adjusting pteval along the way.  Stop when we

                         * detect a page crossing event.

                         */

                        while (sg < sg_end &&

                               (pteval << (64 - IO_PAGE_SHIFT)) != 0UL &&

                               (pteval == SG_ENT_PHYS_ADDRESS(sg)) &&

                               ((pteval ^

                                 (SG_ENT_PHYS_ADDRESS(sg) + sg->length - 1UL)) >> IO_PAGE_SHIFT) == 0UL) {

                                pteval += sg->length;

                                sg++;

                        }

                        if ((pteval << (64 - IO_PAGE_SHIFT)) == 0UL)

                                pteval = ~0UL;

                } while (dma_npages != 0);

                dma_sg++;

        }

}

int sbus_map_sg(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int dir)

{

        struct sbus_iommu *iommu = sdev->bus->iommu;

        unsigned long flags, npages;

        iopte_t *iopte;

        u32 dma_base;

        struct scatterlist *sgtmp;

        int used;

        unsigned long iopte_bits;

        if (dir == SBUS_DMA_NONE)

                BUG();

        /* Fast path single entry scatterlists. */

        if (nents == 1) {

                sg->dma_address =

                        sbus_map_single(sdev,

                                        (sg->address ?

                                         sg->address :

                                         (page_address(sg->page) + sg->offset)),

                                        sg->length, dir);

                sg->dma_length = sg->length;

                return 1;

        }

        npages = prepare_sg(sg, nents);

        spin_lock_irqsave(&iommu->lock, flags);

        iopte = alloc_streaming_cluster(iommu, npages);

        if (iopte == NULL)

                goto bad;

        dma_base = MAP_BASE + ((iopte - iommu->page_table) << IO_PAGE_SHIFT);

        /* Normalize DVMA addresses. */

        sgtmp = sg;

        used = nents;

        while (used && sgtmp->dma_length) {

                sgtmp->dma_address += dma_base;

                sgtmp++;

                used--;

        }

        used = nents - used;

        iopte_bits = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE;

        if (dir != SBUS_DMA_TODEVICE)

                iopte_bits |= IOPTE_WRITE;

        fill_sg(iopte, sg, used, nents, iopte_bits);

#ifdef VERIFY_SG

        verify_sglist(sg, nents, iopte, npages);

#endif

        spin_unlock_irqrestore(&iommu->lock, flags);

        return used;

bad:

        spin_unlock_irqrestore(&iommu->lock, flags);

        BUG();

        return 0;

}

void sbus_unmap_sg(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int direction)

{

        unsigned long size, flags;

        struct sbus_iommu *iommu;

        u32 dvma_base;

        int i;

        /* Fast path single entry scatterlists. */

        if (nents == 1) {

                sbus_unmap_single(sdev, sg->dma_address, sg->dma_length, direction);

                return;

        }

        dvma_base = sg[0].dma_address & IO_PAGE_MASK;

        for (i = 0; i < nents; i++) {

                if (sg[i].dma_length == 0)

                        break;

        }

        i--;

        size = IO_PAGE_ALIGN(sg[i].dma_address + sg[i].dma_length) - dvma_base;

        iommu = sdev->bus->iommu;

        spin_lock_irqsave(&iommu->lock, flags);

        free_streaming_cluster(iommu, dvma_base, size >> IO_PAGE_SHIFT);

        strbuf_flush(iommu, dvma_base, size >> IO_PAGE_SHIFT);

        spin_unlock_irqrestore(&iommu->lock, flags);

}

void sbus_dma_sync_single(struct sbus_dev *sdev, dma_addr_t base, size_t size, int direction)

{

        struct sbus_iommu *iommu = sdev->bus->iommu;

        unsigned long flags;

        size = (IO_PAGE_ALIGN(base + size) - (base & IO_PAGE_MASK));

        spin_lock_irqsave(&iommu->lock, flags);

        strbuf_flush(iommu, base & IO_PAGE_MASK, size >> IO_PAGE_SHIFT);

        spin_unlock_irqrestore(&iommu->lock, flags);

}

void sbus_dma_sync_sg(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int direction)

{

        struct sbus_iommu *iommu = sdev->bus->iommu;

        unsigned long flags, size;

        u32 base;

        int i;

        base = sg[0].dma_address & IO_PAGE_MASK;

        for (i = 0; i < nents; i++) {

                if (sg[i].dma_length == 0)

                        break;

        }

        i--;

        size = IO_PAGE_ALIGN(sg[i].dma_address + sg[i].dma_length) - base;

        spin_lock_irqsave(&iommu->lock, flags);

        strbuf_flush(iommu, base, size >> IO_PAGE_SHIFT);

        spin_unlock_irqrestore(&iommu->lock, flags);

}

/* Enable 64-bit DVMA mode for the given device. */

void sbus_set_sbus64(struct sbus_dev *sdev, int bursts)

{

        struct sbus_iommu *iommu = sdev->bus->iommu;

        int slot = sdev->slot;

        unsigned long cfg_reg;

        u64 val;

        cfg_reg = iommu->sbus_control_reg;

        switch (slot) {

        case 0:

                cfg_reg += 0x20UL;

                break;

        case 1:

                cfg_reg += 0x28UL;

                break;

        case 2:

                cfg_reg += 0x30UL;

                break;

        case 3:

                cfg_reg += 0x38UL;

                break;

        case 13:

                cfg_reg += 0x40UL;

                break;

        case 14:

                cfg_reg += 0x48UL;

                break;

        case 15:

                cfg_reg += 0x50UL;

                break;

        default:

                return;

        };

        val = upa_readq(cfg_reg);

        if (val & (1UL << 14UL)) {

                /* Extended transfer mode already enabled. */

                return;

        }

        val |= (1UL << 14UL);

        if (bursts & DMA_BURST8)

                val |= (1UL << 1UL);

        if (bursts & DMA_BURST16)

                val |= (1UL << 2UL);

        if (bursts & DMA_BURST32)

                val |= (1UL << 3UL);

        if (bursts & DMA_BURST64)

                val |= (1UL << 4UL);

        upa_writeq(val, cfg_reg);

}

/* SBUS SYSIO INO number to Sparc PIL level. */

static unsigned char sysio_ino_to_pil[] = {

        0, 4, 4, 7, 5, 7, 8, 9,          /* SBUS slot 0 */

        0, 4, 4, 7, 5, 7, 8, 9,          /* SBUS slot 1 */

        0, 4, 4, 7, 5, 7, 8, 9,          /* SBUS slot 2 */

        0, 4, 4, 7, 5, 7, 8, 9,          /* SBUS slot 3 */

        4, /* Onboard SCSI */

        5, /* Onboard Ethernet */

/*XXX*/ 8, /* Onboard BPP */

        0, /* Bogon */

       13, /* Audio */

/*XXX*/15, /* PowerFail */