Subversion Repositories test_project

/*

 * Intel I/OAT DMA Linux driver

 * Copyright(c) 2004 - 2007 Intel Corporation.

 *

 * This program is free software; you can redistribute it and/or modify it

 * under the terms and conditions of the GNU General Public License,

 * version 2, as published by the Free Software Foundation.

 *

 * This program is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for

 * more details.

 *

 * You should have received a copy of the GNU General Public License along with

 * this program; if not, write to the Free Software Foundation, Inc.,

 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * The full GNU General Public License is included in this distribution in

 * the file called "COPYING".

 *

 */

/*

 * This driver supports an Intel I/OAT DMA engine, which does asynchronous

 * copy operations.

 */

#include <linux/init.h>

#include <linux/module.h>

#include <linux/pci.h>

#include <linux/interrupt.h>

#include <linux/dmaengine.h>

#include <linux/delay.h>

#include <linux/dma-mapping.h>

#include "ioatdma.h"

#include "ioatdma_registers.h"

#include "ioatdma_hw.h"

#define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common)

#define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, common)

#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)

#define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, async_tx)

static int ioat_pending_level = 4;

module_param(ioat_pending_level, int, 0644);

MODULE_PARM_DESC(ioat_pending_level,

                 "high-water mark for pushing ioat descriptors (default: 4)");

/* internal functions */

static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat_chan);

static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan);

static struct ioat_desc_sw *

ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan);

static struct ioat_desc_sw *

ioat2_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan);

static inline struct ioat_dma_chan *ioat_lookup_chan_by_index(

                                                struct ioatdma_device *device,

                                                int index)

{

        return device->idx[index];

}

/**

 * ioat_dma_do_interrupt - handler used for single vector interrupt mode

 * @irq: interrupt id

 * @data: interrupt data

 */

static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)

{

        struct ioatdma_device *instance = data;

        struct ioat_dma_chan *ioat_chan;

        unsigned long attnstatus;

        int bit;

        u8 intrctrl;

        intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);

        if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))

                return IRQ_NONE;

        if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {

                writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);

                return IRQ_NONE;

        }

        attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);

        for_each_bit(bit, &attnstatus, BITS_PER_LONG) {

                ioat_chan = ioat_lookup_chan_by_index(instance, bit);

                tasklet_schedule(&ioat_chan->cleanup_task);

        }

        writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);

        return IRQ_HANDLED;

}

/**

 * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode

 * @irq: interrupt id

 * @data: interrupt data

 */

static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)

{

        struct ioat_dma_chan *ioat_chan = data;

        tasklet_schedule(&ioat_chan->cleanup_task);

        return IRQ_HANDLED;

}

static void ioat_dma_cleanup_tasklet(unsigned long data);

/**

 * ioat_dma_enumerate_channels - find and initialize the device's channels

 * @device: the device to be enumerated

 */

static int ioat_dma_enumerate_channels(struct ioatdma_device *device)

{

        u8 xfercap_scale;

        u32 xfercap;

        int i;

        struct ioat_dma_chan *ioat_chan;

        device->common.chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);

        xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);

        xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));

        for (i = 0; i < device->common.chancnt; i++) {

                ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL);

                if (!ioat_chan) {

                        device->common.chancnt = i;

                        break;

                }

                ioat_chan->device = device;

                ioat_chan->reg_base = device->reg_base + (0x80 * (i + 1));

                ioat_chan->xfercap = xfercap;

                ioat_chan->desccount = 0;

                if (ioat_chan->device->version != IOAT_VER_1_2) {

                        writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE

                                        | IOAT_DMA_DCA_ANY_CPU,

                                ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);

                }

                spin_lock_init(&ioat_chan->cleanup_lock);

                spin_lock_init(&ioat_chan->desc_lock);

                INIT_LIST_HEAD(&ioat_chan->free_desc);

                INIT_LIST_HEAD(&ioat_chan->used_desc);

                /* This should be made common somewhere in dmaengine.c */

                ioat_chan->common.device = &device->common;

                list_add_tail(&ioat_chan->common.device_node,

                              &device->common.channels);

                device->idx[i] = ioat_chan;

                tasklet_init(&ioat_chan->cleanup_task,

                             ioat_dma_cleanup_tasklet,

                             (unsigned long) ioat_chan);

                tasklet_disable(&ioat_chan->cleanup_task);

        }

        return device->common.chancnt;

}

static void ioat_set_src(dma_addr_t addr,

                         struct dma_async_tx_descriptor *tx,

                         int index)

{

        tx_to_ioat_desc(tx)->src = addr;

}

static void ioat_set_dest(dma_addr_t addr,

                          struct dma_async_tx_descriptor *tx,

                          int index)

{

        tx_to_ioat_desc(tx)->dst = addr;

}

/**

 * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended

 *                                 descriptors to hw

 * @chan: DMA channel handle

 */

static inline void __ioat1_dma_memcpy_issue_pending(

                                                struct ioat_dma_chan *ioat_chan)

{

        ioat_chan->pending = 0;

        writeb(IOAT_CHANCMD_APPEND, ioat_chan->reg_base + IOAT1_CHANCMD_OFFSET);

}

static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan)

{

        struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);

        if (ioat_chan->pending != 0) {

                spin_lock_bh(&ioat_chan->desc_lock);

                __ioat1_dma_memcpy_issue_pending(ioat_chan);

                spin_unlock_bh(&ioat_chan->desc_lock);

        }

}

static inline void __ioat2_dma_memcpy_issue_pending(

                                                struct ioat_dma_chan *ioat_chan)

{

        ioat_chan->pending = 0;

        writew(ioat_chan->dmacount,

               ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);

}

static void ioat2_dma_memcpy_issue_pending(struct dma_chan *chan)

{

        struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);

        if (ioat_chan->pending != 0) {

                spin_lock_bh(&ioat_chan->desc_lock);

                __ioat2_dma_memcpy_issue_pending(ioat_chan);

                spin_unlock_bh(&ioat_chan->desc_lock);

        }

}

static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)

{

        struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);

        struct ioat_desc_sw *first = tx_to_ioat_desc(tx);

        struct ioat_desc_sw *prev, *new;

        struct ioat_dma_descriptor *hw;

        dma_cookie_t cookie;

        LIST_HEAD(new_chain);

        u32 copy;

        size_t len;

        dma_addr_t src, dst;

        int orig_ack;

        unsigned int desc_count = 0;

        /* src and dest and len are stored in the initial descriptor */

        len = first->len;

        src = first->src;

        dst = first->dst;

        orig_ack = first->async_tx.ack;

        new = first;

        spin_lock_bh(&ioat_chan->desc_lock);

        prev = to_ioat_desc(ioat_chan->used_desc.prev);

        prefetch(prev->hw);

        do {

                copy = min_t(size_t, len, ioat_chan->xfercap);

                new->async_tx.ack = 1;

                hw = new->hw;

                hw->size = copy;

                hw->ctl = 0;

                hw->src_addr = src;

                hw->dst_addr = dst;

                hw->next = 0;

                /* chain together the physical address list for the HW */

                wmb();

                prev->hw->next = (u64) new->async_tx.phys;

                len -= copy;

                dst += copy;

                src += copy;

                list_add_tail(&new->node, &new_chain);

                desc_count++;

                prev = new;

        } while (len && (new = ioat1_dma_get_next_descriptor(ioat_chan)));

        hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;

        if (new->async_tx.callback) {

                hw->ctl |= IOAT_DMA_DESCRIPTOR_CTL_INT_GN;

                if (first != new) {

                        /* move callback into to last desc */

                        new->async_tx.callback = first->async_tx.callback;

                        new->async_tx.callback_param

                                        = first->async_tx.callback_param;

                        first->async_tx.callback = NULL;

                        first->async_tx.callback_param = NULL;

                }

        }

        new->tx_cnt = desc_count;

        new->async_tx.ack = orig_ack; /* client is in control of this ack */

        /* store the original values for use in later cleanup */

        if (new != first) {

                new->src = first->src;

                new->dst = first->dst;

                new->len = first->len;

        }

        /* cookie incr and addition to used_list must be atomic */

        cookie = ioat_chan->common.cookie;

        cookie++;

        if (cookie < 0)

                cookie = 1;

        ioat_chan->common.cookie = new->async_tx.cookie = cookie;

        /* write address into NextDescriptor field of last desc in chain */

        to_ioat_desc(ioat_chan->used_desc.prev)->hw->next =

                                                        first->async_tx.phys;

        __list_splice(&new_chain, ioat_chan->used_desc.prev);

        ioat_chan->dmacount += desc_count;

        ioat_chan->pending += desc_count;

        if (ioat_chan->pending >= ioat_pending_level)

                __ioat1_dma_memcpy_issue_pending(ioat_chan);

        spin_unlock_bh(&ioat_chan->desc_lock);

        return cookie;

}

static dma_cookie_t ioat2_tx_submit(struct dma_async_tx_descriptor *tx)

{

        struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);

        struct ioat_desc_sw *first = tx_to_ioat_desc(tx);

        struct ioat_desc_sw *new;

        struct ioat_dma_descriptor *hw;

        dma_cookie_t cookie;

        u32 copy;

        size_t len;

        dma_addr_t src, dst;

        int orig_ack;

        unsigned int desc_count = 0;

        /* src and dest and len are stored in the initial descriptor */

        len = first->len;

        src = first->src;

        dst = first->dst;

        orig_ack = first->async_tx.ack;

        new = first;

        /*

         * ioat_chan->desc_lock is still in force in version 2 path

         * it gets unlocked at end of this function

         */

        do {

                copy = min_t(size_t, len, ioat_chan->xfercap);

                new->async_tx.ack = 1;

                hw = new->hw;

                hw->size = copy;

                hw->ctl = 0;

                hw->src_addr = src;

                hw->dst_addr = dst;

                len -= copy;

                dst += copy;

                src += copy;

                desc_count++;

        } while (len && (new = ioat2_dma_get_next_descriptor(ioat_chan)));

        hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;

        if (new->async_tx.callback) {

                hw->ctl |= IOAT_DMA_DESCRIPTOR_CTL_INT_GN;

                if (first != new) {

                        /* move callback into to last desc */

                        new->async_tx.callback = first->async_tx.callback;

                        new->async_tx.callback_param

                                        = first->async_tx.callback_param;

                        first->async_tx.callback = NULL;

                        first->async_tx.callback_param = NULL;

                }

        }

        new->tx_cnt = desc_count;

        new->async_tx.ack = orig_ack; /* client is in control of this ack */

        /* store the original values for use in later cleanup */

        if (new != first) {

                new->src = first->src;

                new->dst = first->dst;

                new->len = first->len;

        }

        /* cookie incr and addition to used_list must be atomic */

        cookie = ioat_chan->common.cookie;

        cookie++;

        if (cookie < 0)

                cookie = 1;

        ioat_chan->common.cookie = new->async_tx.cookie = cookie;

        ioat_chan->dmacount += desc_count;

        ioat_chan->pending += desc_count;

        if (ioat_chan->pending >= ioat_pending_level)

                __ioat2_dma_memcpy_issue_pending(ioat_chan);

        spin_unlock_bh(&ioat_chan->desc_lock);

        return cookie;

}

/**

 * ioat_dma_alloc_descriptor - allocate and return a sw and hw descriptor pair

 * @ioat_chan: the channel supplying the memory pool for the descriptors

 * @flags: allocation flags

 */

static struct ioat_desc_sw *ioat_dma_alloc_descriptor(

                                        struct ioat_dma_chan *ioat_chan,

                                        gfp_t flags)

{

        struct ioat_dma_descriptor *desc;

        struct ioat_desc_sw *desc_sw;

        struct ioatdma_device *ioatdma_device;

        dma_addr_t phys;

        ioatdma_device = to_ioatdma_device(ioat_chan->common.device);

        desc = pci_pool_alloc(ioatdma_device->dma_pool, flags, &phys);

        if (unlikely(!desc))

                return NULL;

        desc_sw = kzalloc(sizeof(*desc_sw), flags);

        if (unlikely(!desc_sw)) {

                pci_pool_free(ioatdma_device->dma_pool, desc, phys);

                return NULL;

        }

        memset(desc, 0, sizeof(*desc));

        dma_async_tx_descriptor_init(&desc_sw->async_tx, &ioat_chan->common);

        desc_sw->async_tx.tx_set_src = ioat_set_src;

        desc_sw->async_tx.tx_set_dest = ioat_set_dest;

        switch (ioat_chan->device->version) {

        case IOAT_VER_1_2:

                desc_sw->async_tx.tx_submit = ioat1_tx_submit;

                break;

        case IOAT_VER_2_0:

                desc_sw->async_tx.tx_submit = ioat2_tx_submit;

                break;

        }

        INIT_LIST_HEAD(&desc_sw->async_tx.tx_list);

        desc_sw->hw = desc;

        desc_sw->async_tx.phys = phys;

        return desc_sw;

}

static int ioat_initial_desc_count = 256;

module_param(ioat_initial_desc_count, int, 0644);

MODULE_PARM_DESC(ioat_initial_desc_count,

                 "initial descriptors per channel (default: 256)");

/**

 * ioat2_dma_massage_chan_desc - link the descriptors into a circle

 * @ioat_chan: the channel to be massaged

 */

static void ioat2_dma_massage_chan_desc(struct ioat_dma_chan *ioat_chan)

{

        struct ioat_desc_sw *desc, *_desc;

        /* setup used_desc */

        ioat_chan->used_desc.next = ioat_chan->free_desc.next;

        ioat_chan->used_desc.prev = NULL;

        /* pull free_desc out of the circle so that every node is a hw

         * descriptor, but leave it pointing to the list

         */

        ioat_chan->free_desc.prev->next = ioat_chan->free_desc.next;

        ioat_chan->free_desc.next->prev = ioat_chan->free_desc.prev;

        /* circle link the hw descriptors */

        desc = to_ioat_desc(ioat_chan->free_desc.next);

        desc->hw->next = to_ioat_desc(desc->node.next)->async_tx.phys;

        list_for_each_entry_safe(desc, _desc, ioat_chan->free_desc.next, node) {

                desc->hw->next = to_ioat_desc(desc->node.next)->async_tx.phys;

        }

}

/**

 * ioat_dma_alloc_chan_resources - returns the number of allocated descriptors

 * @chan: the channel to be filled out

 */

static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)

{

        struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);

        struct ioat_desc_sw *desc;

        u16 chanctrl;

        u32 chanerr;

        int i;

        LIST_HEAD(tmp_list);

        /* have we already been set up? */

        if (!list_empty(&ioat_chan->free_desc))

                return ioat_chan->desccount;

        /* Setup register to interrupt and write completion status on error */

        chanctrl = IOAT_CHANCTRL_ERR_INT_EN |

                IOAT_CHANCTRL_ANY_ERR_ABORT_EN |

                IOAT_CHANCTRL_ERR_COMPLETION_EN;

        writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);

        chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);

        if (chanerr) {

                dev_err(&ioat_chan->device->pdev->dev,

                        "CHANERR = %x, clearing\n", chanerr);

                writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);

        }

        /* Allocate descriptors */

        for (i = 0; i < ioat_initial_desc_count; i++) {

                desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL);

                if (!desc) {

                        dev_err(&ioat_chan->device->pdev->dev,

                                "Only %d initial descriptors\n", i);

                        break;

                }

                list_add_tail(&desc->node, &tmp_list);

        }

        spin_lock_bh(&ioat_chan->desc_lock);

        ioat_chan->desccount = i;

        list_splice(&tmp_list, &ioat_chan->free_desc);

        if (ioat_chan->device->version != IOAT_VER_1_2)

                ioat2_dma_massage_chan_desc(ioat_chan);

        spin_unlock_bh(&ioat_chan->desc_lock);

        /* allocate a completion writeback area */

        /* doing 2 32bit writes to mmio since 1 64b write doesn't work */

        ioat_chan->completion_virt =

                pci_pool_alloc(ioat_chan->device->completion_pool,

                               GFP_KERNEL,

                               &ioat_chan->completion_addr);

        memset(ioat_chan->completion_virt, 0,

               sizeof(*ioat_chan->completion_virt));

        writel(((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF,

               ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);

        writel(((u64) ioat_chan->completion_addr) >> 32,

               ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);

        tasklet_enable(&ioat_chan->cleanup_task);

        ioat_dma_start_null_desc(ioat_chan);  /* give chain to dma device */

        return ioat_chan->desccount;

}

/**

 * ioat_dma_free_chan_resources - release all the descriptors

 * @chan: the channel to be cleaned

 */

static void ioat_dma_free_chan_resources(struct dma_chan *chan)

{

        struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);

        struct ioatdma_device *ioatdma_device = to_ioatdma_device(chan->device);

        struct ioat_desc_sw *desc, *_desc;

        int in_use_descs = 0;

        tasklet_disable(&ioat_chan->cleanup_task);

        ioat_dma_memcpy_cleanup(ioat_chan);

        /* Delay 100ms after reset to allow internal DMA logic to quiesce

         * before removing DMA descriptor resources.

         */

        writeb(IOAT_CHANCMD_RESET,

               ioat_chan->reg_base

                        + IOAT_CHANCMD_OFFSET(ioat_chan->device->version));

        mdelay(100);

        spin_lock_bh(&ioat_chan->desc_lock);

        switch (ioat_chan->device->version) {

        case IOAT_VER_1_2:

                list_for_each_entry_safe(desc, _desc,

                                         &ioat_chan->used_desc, node) {

                        in_use_descs++;

                        list_del(&desc->node);

                        pci_pool_free(ioatdma_device->dma_pool, desc->hw,

                                      desc->async_tx.phys);

                        kfree(desc);

                }

                list_for_each_entry_safe(desc, _desc,

                                         &ioat_chan->free_desc, node) {

                        list_del(&desc->node);

                        pci_pool_free(ioatdma_device->dma_pool, desc->hw,

                                      desc->async_tx.phys);

                        kfree(desc);

                }

                break;

        case IOAT_VER_2_0:

                list_for_each_entry_safe(desc, _desc,

                                         ioat_chan->free_desc.next, node) {

                        list_del(&desc->node);

                        pci_pool_free(ioatdma_device->dma_pool, desc->hw,

                                      desc->async_tx.phys);

                        kfree(desc);

                }

                desc = to_ioat_desc(ioat_chan->free_desc.next);

                pci_pool_free(ioatdma_device->dma_pool, desc->hw,

                              desc->async_tx.phys);

                kfree(desc);

                INIT_LIST_HEAD(&ioat_chan->free_desc);

                INIT_LIST_HEAD(&ioat_chan->used_desc);

                break;

        }

        spin_unlock_bh(&ioat_chan->desc_lock);

        pci_pool_free(ioatdma_device->completion_pool,

                      ioat_chan->completion_virt,

                      ioat_chan->completion_addr);

        /* one is ok since we left it on there on purpose */

        if (in_use_descs > 1)

                dev_err(&ioat_chan->device->pdev->dev,

                        "Freeing %d in use descriptors!\n",

                        in_use_descs - 1);

        ioat_chan->last_completion = ioat_chan->completion_addr = 0;

        ioat_chan->pending = 0;

        ioat_chan->dmacount = 0;

}

/**

 * ioat_dma_get_next_descriptor - return the next available descriptor

 * @ioat_chan: IOAT DMA channel handle

 *

 * Gets the next descriptor from the chain, and must be called with the

 * channel's desc_lock held.  Allocates more descriptors if the channel

 * has run out.

 */

static struct ioat_desc_sw *

ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan)

{

        struct ioat_desc_sw *new;

        if (!list_empty(&ioat_chan->free_desc)) {

                new = to_ioat_desc(ioat_chan->free_desc.next);

                list_del(&new->node);

        } else {

                /* try to get another desc */

                new = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC);

                if (!new) {

                        dev_err(&ioat_chan->device->pdev->dev,

                                "alloc failed\n");

                        return NULL;

                }

        }

        prefetch(new->hw);

        return new;

}

static struct ioat_desc_sw *

ioat2_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan)

{

        struct ioat_desc_sw *new;

        /*

         * used.prev points to where to start processing

         * used.next points to next free descriptor

         * if used.prev == NULL, there are none waiting to be processed

         * if used.next == used.prev.prev, there is only one free descriptor,

         *      and we need to use it to as a noop descriptor before

         *      linking in a new set of descriptors, since the device

         *      has probably already read the pointer to it

         */

        if (ioat_chan->used_desc.prev &&

            ioat_chan->used_desc.next == ioat_chan->used_desc.prev->prev) {

                struct ioat_desc_sw *desc;

                struct ioat_desc_sw *noop_desc;

                int i;

                /* set up the noop descriptor */

                noop_desc = to_ioat_desc(ioat_chan->used_desc.next);

                noop_desc->hw->size = 0;

                noop_desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL;

                noop_desc->hw->src_addr = 0;

                noop_desc->hw->dst_addr = 0;

                ioat_chan->used_desc.next = ioat_chan->used_desc.next->next;

                ioat_chan->pending++;

                ioat_chan->dmacount++;

                /* try to get a few more descriptors */

                for (i = 16; i; i--) {

                        desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC);

                        if (!desc) {

                                dev_err(&ioat_chan->device->pdev->dev,

                                        "alloc failed\n");

                                break;

                        }