URL https://opencores.org/ocsvn/or1k_soc_on_altera_embedded_dev_kit/or1k_soc_on_altera_embedded_dev_kit/trunk

Subversion Repositories or1k_soc_on_altera_embedded_dev_kit

[/] [or1k_soc_on_altera_embedded_dev_kit/] [trunk/] [linux-2.6/] [linux-2.6.24/] [net/] [ipv4/] [inet_lro.c] - Blame information for rev 17

Go to most recent revision | Details | Compare with Previous | View Log


/*
 *  linux/net/ipv4/inet_lro.c
 *
 *  Large Receive Offload (ipv4 / tcp)
 *
 *  (C) Copyright IBM Corp. 2007
 *
 *  Authors:
 *       Jan-Bernd Themann <themann@de.ibm.com>
 *       Christoph Raisch <raisch@de.ibm.com>
 *
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2, or (at your option)
 * any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */
 
 
#include <linux/module.h>
#include <linux/if_vlan.h>
#include <linux/inet_lro.h>
 
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Jan-Bernd Themann <themann@de.ibm.com>");
MODULE_DESCRIPTION("Large Receive Offload (ipv4 / tcp)");
 
#define TCP_HDR_LEN(tcph) (tcph->doff << 2)
#define IP_HDR_LEN(iph) (iph->ihl << 2)
#define TCP_PAYLOAD_LENGTH(iph, tcph) \
        (ntohs(iph->tot_len) - IP_HDR_LEN(iph) - TCP_HDR_LEN(tcph))
 
#define IPH_LEN_WO_OPTIONS 5
#define TCPH_LEN_WO_OPTIONS 5
#define TCPH_LEN_W_TIMESTAMP 8
 
#define LRO_MAX_PG_HLEN 64
 
#define LRO_INC_STATS(lro_mgr, attr) { lro_mgr->stats.attr++; }
 
/*
 * Basic tcp checks whether packet is suitable for LRO
 */
 
static int lro_tcp_ip_check(struct iphdr *iph, struct tcphdr *tcph,
                            int len, struct net_lro_desc *lro_desc)
{
        /* check ip header: don't aggregate padded frames */
        if (ntohs(iph->tot_len) != len)
                return -1;
 
        if (TCP_PAYLOAD_LENGTH(iph, tcph) == 0)
                return -1;
 
        if (iph->ihl != IPH_LEN_WO_OPTIONS)
                return -1;
 
        if (tcph->cwr || tcph->ece || tcph->urg || !tcph->ack
            || tcph->rst || tcph->syn || tcph->fin)
                return -1;
 
        if (INET_ECN_is_ce(ipv4_get_dsfield(iph)))
                return -1;
 
        if (tcph->doff != TCPH_LEN_WO_OPTIONS
            && tcph->doff != TCPH_LEN_W_TIMESTAMP)
                return -1;
 
        /* check tcp options (only timestamp allowed) */
        if (tcph->doff == TCPH_LEN_W_TIMESTAMP) {
                __be32 *topt = (__be32 *)(tcph + 1);
 
                if (*topt != htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16)
                                   | (TCPOPT_TIMESTAMP << 8)
                                   | TCPOLEN_TIMESTAMP))
                        return -1;
 
                /* timestamp should be in right order */
                topt++;
                if (lro_desc && after(ntohl(lro_desc->tcp_rcv_tsval),
                                      ntohl(*topt)))
                        return -1;
 
                /* timestamp reply should not be zero */
                topt++;
                if (*topt == 0)
                        return -1;
        }
 
        return 0;
}
 
static void lro_update_tcp_ip_header(struct net_lro_desc *lro_desc)
{
        struct iphdr *iph = lro_desc->iph;
        struct tcphdr *tcph = lro_desc->tcph;
        __be32 *p;
        __wsum tcp_hdr_csum;
 
        tcph->ack_seq = lro_desc->tcp_ack;
        tcph->window = lro_desc->tcp_window;
 
        if (lro_desc->tcp_saw_tstamp) {
                p = (__be32 *)(tcph + 1);
                *(p+2) = lro_desc->tcp_rcv_tsecr;
        }
 
        iph->tot_len = htons(lro_desc->ip_tot_len);
 
        iph->check = 0;
        iph->check = ip_fast_csum((u8 *)lro_desc->iph, iph->ihl);
 
        tcph->check = 0;
        tcp_hdr_csum = csum_partial((u8 *)tcph, TCP_HDR_LEN(tcph), 0);
        lro_desc->data_csum = csum_add(lro_desc->data_csum, tcp_hdr_csum);
        tcph->check = csum_tcpudp_magic(iph->saddr, iph->daddr,
                                        lro_desc->ip_tot_len -
                                        IP_HDR_LEN(iph), IPPROTO_TCP,
                                        lro_desc->data_csum);
}
 
static __wsum lro_tcp_data_csum(struct iphdr *iph, struct tcphdr *tcph, int len)
{
        __wsum tcp_csum;
        __wsum tcp_hdr_csum;
        __wsum tcp_ps_hdr_csum;
 
        tcp_csum = ~csum_unfold(tcph->check);
        tcp_hdr_csum = csum_partial((u8 *)tcph, TCP_HDR_LEN(tcph), tcp_csum);
 
        tcp_ps_hdr_csum = csum_tcpudp_nofold(iph->saddr, iph->daddr,
                                             len + TCP_HDR_LEN(tcph),
                                             IPPROTO_TCP, 0);
 
        return csum_sub(csum_sub(tcp_csum, tcp_hdr_csum),
                        tcp_ps_hdr_csum);
}
 
static void lro_init_desc(struct net_lro_desc *lro_desc, struct sk_buff *skb,
                          struct iphdr *iph, struct tcphdr *tcph,
                          u16 vlan_tag, struct vlan_group *vgrp)
{
        int nr_frags;
        __be32 *ptr;
        u32 tcp_data_len = TCP_PAYLOAD_LENGTH(iph, tcph);
 
        nr_frags = skb_shinfo(skb)->nr_frags;
        lro_desc->parent = skb;
        lro_desc->next_frag = &(skb_shinfo(skb)->frags[nr_frags]);
        lro_desc->iph = iph;
        lro_desc->tcph = tcph;
        lro_desc->tcp_next_seq = ntohl(tcph->seq) + tcp_data_len;
        lro_desc->tcp_ack = tcph->ack_seq;
        lro_desc->tcp_window = tcph->window;
 
        lro_desc->pkt_aggr_cnt = 1;
        lro_desc->ip_tot_len = ntohs(iph->tot_len);
 
        if (tcph->doff == 8) {
                ptr = (__be32 *)(tcph+1);
                lro_desc->tcp_saw_tstamp = 1;
                lro_desc->tcp_rcv_tsval = *(ptr+1);
                lro_desc->tcp_rcv_tsecr = *(ptr+2);
        }
 
        lro_desc->mss = tcp_data_len;
        lro_desc->vgrp = vgrp;
        lro_desc->vlan_tag = vlan_tag;
        lro_desc->active = 1;
 
        lro_desc->data_csum = lro_tcp_data_csum(iph, tcph,
                                                tcp_data_len);
}
 
static inline void lro_clear_desc(struct net_lro_desc *lro_desc)
{
        memset(lro_desc, 0, sizeof(struct net_lro_desc));
}
 
static void lro_add_common(struct net_lro_desc *lro_desc, struct iphdr *iph,
                           struct tcphdr *tcph, int tcp_data_len)
{
        struct sk_buff *parent = lro_desc->parent;
        __be32 *topt;
 
        lro_desc->pkt_aggr_cnt++;
        lro_desc->ip_tot_len += tcp_data_len;
        lro_desc->tcp_next_seq += tcp_data_len;
        lro_desc->tcp_window = tcph->window;
        lro_desc->tcp_ack = tcph->ack_seq;
 
        /* don't update tcp_rcv_tsval, would not work with PAWS */
        if (lro_desc->tcp_saw_tstamp) {
                topt = (__be32 *) (tcph + 1);
                lro_desc->tcp_rcv_tsecr = *(topt + 2);
        }
 
        lro_desc->data_csum = csum_block_add(lro_desc->data_csum,
                                             lro_tcp_data_csum(iph, tcph,
                                                               tcp_data_len),
                                             parent->len);
 
        parent->len += tcp_data_len;
        parent->data_len += tcp_data_len;
        if (tcp_data_len > lro_desc->mss)
                lro_desc->mss = tcp_data_len;
}
 
static void lro_add_packet(struct net_lro_desc *lro_desc, struct sk_buff *skb,
                           struct iphdr *iph, struct tcphdr *tcph)
{
        struct sk_buff *parent = lro_desc->parent;
        int tcp_data_len = TCP_PAYLOAD_LENGTH(iph, tcph);
 
        lro_add_common(lro_desc, iph, tcph, tcp_data_len);
 
        skb_pull(skb, (skb->len - tcp_data_len));
        parent->truesize += skb->truesize;
 
        if (lro_desc->last_skb)
                lro_desc->last_skb->next = skb;
        else
                skb_shinfo(parent)->frag_list = skb;
 
        lro_desc->last_skb = skb;
}
 
static void lro_add_frags(struct net_lro_desc *lro_desc,
                          int len, int hlen, int truesize,
                          struct skb_frag_struct *skb_frags,
                          struct iphdr *iph, struct tcphdr *tcph)
{
        struct sk_buff *skb = lro_desc->parent;
        int tcp_data_len = TCP_PAYLOAD_LENGTH(iph, tcph);
 
        lro_add_common(lro_desc, iph, tcph, tcp_data_len);
 
        skb->truesize += truesize;
 
        skb_frags[0].page_offset += hlen;
        skb_frags[0].size -= hlen;
 
        while (tcp_data_len > 0) {
                *(lro_desc->next_frag) = *skb_frags;
                tcp_data_len -= skb_frags->size;
                lro_desc->next_frag++;
                skb_frags++;
                skb_shinfo(skb)->nr_frags++;
        }
}
 
static int lro_check_tcp_conn(struct net_lro_desc *lro_desc,
                              struct iphdr *iph,
                              struct tcphdr *tcph)
{
        if ((lro_desc->iph->saddr != iph->saddr)
            || (lro_desc->iph->daddr != iph->daddr)
            || (lro_desc->tcph->source != tcph->source)
            || (lro_desc->tcph->dest != tcph->dest))
                return -1;
        return 0;
}
 
static struct net_lro_desc *lro_get_desc(struct net_lro_mgr *lro_mgr,
                                         struct net_lro_desc *lro_arr,
                                         struct iphdr *iph,
                                         struct tcphdr *tcph)
{
        struct net_lro_desc *lro_desc = NULL;
        struct net_lro_desc *tmp;
        int max_desc = lro_mgr->max_desc;
        int i;
 
        for (i = 0; i < max_desc; i++) {
                tmp = &lro_arr[i];
                if (tmp->active)
                        if (!lro_check_tcp_conn(tmp, iph, tcph)) {
                                lro_desc = tmp;
                                goto out;
                        }
        }
 
        for (i = 0; i < max_desc; i++) {
                if (!lro_arr[i].active) {
                        lro_desc = &lro_arr[i];
                        goto out;
                }
        }
 
        LRO_INC_STATS(lro_mgr, no_desc);
out:
        return lro_desc;
}
 
static void lro_flush(struct net_lro_mgr *lro_mgr,
                      struct net_lro_desc *lro_desc)
{
        if (lro_desc->pkt_aggr_cnt > 1)
                lro_update_tcp_ip_header(lro_desc);
 
        skb_shinfo(lro_desc->parent)->gso_size = lro_desc->mss;
 
        if (lro_desc->vgrp) {
                if (lro_mgr->features & LRO_F_NAPI)
                        vlan_hwaccel_receive_skb(lro_desc->parent,
                                                 lro_desc->vgrp,
                                                 lro_desc->vlan_tag);
                else
                        vlan_hwaccel_rx(lro_desc->parent,
                                        lro_desc->vgrp,
                                        lro_desc->vlan_tag);
 
        } else {
                if (lro_mgr->features & LRO_F_NAPI)
                        netif_receive_skb(lro_desc->parent);
                else
                        netif_rx(lro_desc->parent);
        }
 
        LRO_INC_STATS(lro_mgr, flushed);
        lro_clear_desc(lro_desc);
}
 
static int __lro_proc_skb(struct net_lro_mgr *lro_mgr, struct sk_buff *skb,
                          struct vlan_group *vgrp, u16 vlan_tag, void *priv)
{
        struct net_lro_desc *lro_desc;
        struct iphdr *iph;
        struct tcphdr *tcph;
        u64 flags;
        int vlan_hdr_len = 0;
 
        if (!lro_mgr->get_skb_header
            || lro_mgr->get_skb_header(skb, (void *)&iph, (void *)&tcph,
                                       &flags, priv))
                goto out;
 
        if (!(flags & LRO_IPV4) || !(flags & LRO_TCP))
                goto out;
 
        lro_desc = lro_get_desc(lro_mgr, lro_mgr->lro_arr, iph, tcph);
        if (!lro_desc)
                goto out;
 
        if ((skb->protocol == htons(ETH_P_8021Q))
            && !(lro_mgr->features & LRO_F_EXTRACT_VLAN_ID))
                vlan_hdr_len = VLAN_HLEN;
 
        if (!lro_desc->active) { /* start new lro session */
                if (lro_tcp_ip_check(iph, tcph, skb->len - vlan_hdr_len, NULL))
                        goto out;
 
                skb->ip_summed = lro_mgr->ip_summed_aggr;
                lro_init_desc(lro_desc, skb, iph, tcph, vlan_tag, vgrp);
                LRO_INC_STATS(lro_mgr, aggregated);
                return 0;
        }
 
        if (lro_desc->tcp_next_seq != ntohl(tcph->seq))
                goto out2;
 
        if (lro_tcp_ip_check(iph, tcph, skb->len, lro_desc))
                goto out2;
 
        lro_add_packet(lro_desc, skb, iph, tcph);
        LRO_INC_STATS(lro_mgr, aggregated);
 
        if ((lro_desc->pkt_aggr_cnt >= lro_mgr->max_aggr) ||
            lro_desc->parent->len > (0xFFFF - lro_mgr->dev->mtu))
                lro_flush(lro_mgr, lro_desc);
 
        return 0;
 
out2: /* send aggregated SKBs to stack */
        lro_flush(lro_mgr, lro_desc);
 
out:  /* Original SKB has to be posted to stack */
        skb->ip_summed = lro_mgr->ip_summed;
        return 1;
}
 
 
static struct sk_buff *lro_gen_skb(struct net_lro_mgr *lro_mgr,
                                   struct skb_frag_struct *frags,
                                   int len, int true_size,
                                   void *mac_hdr,
                                   int hlen, __wsum sum,
                                   u32 ip_summed)
{
        struct sk_buff *skb;
        struct skb_frag_struct *skb_frags;
        int data_len = len;
        int hdr_len = min(len, hlen);
 
        skb = netdev_alloc_skb(lro_mgr->dev, hlen + lro_mgr->frag_align_pad);
        if (!skb)
                return NULL;
 
        skb_reserve(skb, lro_mgr->frag_align_pad);
        skb->len = len;
        skb->data_len = len - hdr_len;
        skb->truesize += true_size;
        skb->tail += hdr_len;
 
        memcpy(skb->data, mac_hdr, hdr_len);
 
        skb_frags = skb_shinfo(skb)->frags;
        while (data_len > 0) {
                *skb_frags = *frags;
                data_len -= frags->size;
                skb_frags++;
                frags++;
                skb_shinfo(skb)->nr_frags++;
        }
 
        skb_shinfo(skb)->frags[0].page_offset += hdr_len;
        skb_shinfo(skb)->frags[0].size -= hdr_len;
 
        skb->ip_summed = ip_summed;
        skb->csum = sum;
        skb->protocol = eth_type_trans(skb, lro_mgr->dev);
        return skb;
}
 
static struct sk_buff *__lro_proc_segment(struct net_lro_mgr *lro_mgr,
                                          struct skb_frag_struct *frags,
                                          int len, int true_size,
                                          struct vlan_group *vgrp,
                                          u16 vlan_tag, void *priv, __wsum sum)
{
        struct net_lro_desc *lro_desc;
        struct iphdr *iph;
        struct tcphdr *tcph;
        struct sk_buff *skb;
        u64 flags;
        void *mac_hdr;
        int mac_hdr_len;
        int hdr_len = LRO_MAX_PG_HLEN;
        int vlan_hdr_len = 0;
 
        if (!lro_mgr->get_frag_header
            || lro_mgr->get_frag_header(frags, (void *)&mac_hdr, (void *)&iph,
                                        (void *)&tcph, &flags, priv)) {
                mac_hdr = page_address(frags->page) + frags->page_offset;
                goto out1;
        }
 
        if (!(flags & LRO_IPV4) || !(flags & LRO_TCP))
                goto out1;
 
        hdr_len = (int)((void *)(tcph) + TCP_HDR_LEN(tcph) - mac_hdr);
        mac_hdr_len = (int)((void *)(iph) - mac_hdr);
 
        lro_desc = lro_get_desc(lro_mgr, lro_mgr->lro_arr, iph, tcph);
        if (!lro_desc)
                goto out1;
 
        if (!lro_desc->active) { /* start new lro session */
                if (lro_tcp_ip_check(iph, tcph, len - mac_hdr_len, NULL))
                        goto out1;
 
                skb = lro_gen_skb(lro_mgr, frags, len, true_size, mac_hdr,
                                  hdr_len, 0, lro_mgr->ip_summed_aggr);
                if (!skb)
                        goto out;
 
                if ((skb->protocol == htons(ETH_P_8021Q))
                    && !(lro_mgr->features & LRO_F_EXTRACT_VLAN_ID))
                        vlan_hdr_len = VLAN_HLEN;
 
                iph = (void *)(skb->data + vlan_hdr_len);
                tcph = (void *)((u8 *)skb->data + vlan_hdr_len
                                + IP_HDR_LEN(iph));
 
                lro_init_desc(lro_desc, skb, iph, tcph, 0, NULL);
                LRO_INC_STATS(lro_mgr, aggregated);
                return NULL;
        }
 
        if (lro_desc->tcp_next_seq != ntohl(tcph->seq))
                goto out2;
 
        if (lro_tcp_ip_check(iph, tcph, len - mac_hdr_len, lro_desc))
                goto out2;
 
        lro_add_frags(lro_desc, len, hdr_len, true_size, frags, iph, tcph);
        LRO_INC_STATS(lro_mgr, aggregated);
 
        if ((skb_shinfo(lro_desc->parent)->nr_frags >= lro_mgr->max_aggr) ||
            lro_desc->parent->len > (0xFFFF - lro_mgr->dev->mtu))
                lro_flush(lro_mgr, lro_desc);
 
        return NULL;
 
out2: /* send aggregated packets to the stack */
        lro_flush(lro_mgr, lro_desc);
 
out1:  /* Original packet has to be posted to the stack */
        skb = lro_gen_skb(lro_mgr, frags, len, true_size, mac_hdr,
                          hdr_len, sum, lro_mgr->ip_summed);
out:
        return skb;
}
 
void lro_receive_skb(struct net_lro_mgr *lro_mgr,
                     struct sk_buff *skb,
                     void *priv)
{
        if (__lro_proc_skb(lro_mgr, skb, NULL, 0, priv)) {
                if (lro_mgr->features & LRO_F_NAPI)
                        netif_receive_skb(skb);
                else
                        netif_rx(skb);
        }
}
EXPORT_SYMBOL(lro_receive_skb);
 
void lro_vlan_hwaccel_receive_skb(struct net_lro_mgr *lro_mgr,
                                  struct sk_buff *skb,
                                  struct vlan_group *vgrp,
                                  u16 vlan_tag,
                                  void *priv)
{
        if (__lro_proc_skb(lro_mgr, skb, vgrp, vlan_tag, priv)) {
                if (lro_mgr->features & LRO_F_NAPI)
                        vlan_hwaccel_receive_skb(skb, vgrp, vlan_tag);
                else
                        vlan_hwaccel_rx(skb, vgrp, vlan_tag);
        }
}
EXPORT_SYMBOL(lro_vlan_hwaccel_receive_skb);
 
void lro_receive_frags(struct net_lro_mgr *lro_mgr,
                       struct skb_frag_struct *frags,
                       int len, int true_size, void *priv, __wsum sum)
{
        struct sk_buff *skb;
 
        skb = __lro_proc_segment(lro_mgr, frags, len, true_size, NULL, 0,
                                 priv, sum);
        if (!skb)
                return;
 
        if (lro_mgr->features & LRO_F_NAPI)
                netif_receive_skb(skb);
        else
                netif_rx(skb);
}
EXPORT_SYMBOL(lro_receive_frags);
 
void lro_vlan_hwaccel_receive_frags(struct net_lro_mgr *lro_mgr,
                                    struct skb_frag_struct *frags,
                                    int len, int true_size,
                                    struct vlan_group *vgrp,
                                    u16 vlan_tag, void *priv, __wsum sum)
{
        struct sk_buff *skb;
 
        skb = __lro_proc_segment(lro_mgr, frags, len, true_size, vgrp,
                                 vlan_tag, priv, sum);
        if (!skb)
                return;
 
        if (lro_mgr->features & LRO_F_NAPI)
                vlan_hwaccel_receive_skb(skb, vgrp, vlan_tag);
        else
                vlan_hwaccel_rx(skb, vgrp, vlan_tag);
}
EXPORT_SYMBOL(lro_vlan_hwaccel_receive_frags);
 
void lro_flush_all(struct net_lro_mgr *lro_mgr)
{
        int i;
        struct net_lro_desc *lro_desc = lro_mgr->lro_arr;
 
        for (i = 0; i < lro_mgr->max_desc; i++) {
                if (lro_desc[i].active)
                        lro_flush(lro_mgr, &lro_desc[i]);
        }
}
EXPORT_SYMBOL(lro_flush_all);
 
void lro_flush_pkt(struct net_lro_mgr *lro_mgr,
                  struct iphdr *iph, struct tcphdr *tcph)
{
        struct net_lro_desc *lro_desc;
 
        lro_desc = lro_get_desc(lro_mgr, lro_mgr->lro_arr, iph, tcph);
        if (lro_desc->active)
                lro_flush(lro_mgr, lro_desc);
}
EXPORT_SYMBOL(lro_flush_pkt);

Browse

Tools

Subversion Repositories or1k_soc_on_altera_embedded_dev_kit

[/] [or1k_soc_on_altera_embedded_dev_kit/] [trunk/] [linux-2.6/] [linux-2.6.24/] [net/] [ipv4/] [inet_lro.c] - Blame information for rev 17

Line No.	Rev	Author	Line
1	3	xianfeng	`/*`
2			`* linux/net/ipv4/inet_lro.c`
3			`*`
4			`* Large Receive Offload (ipv4 / tcp)`
5			`*`
6			`* (C) Copyright IBM Corp. 2007`
7			`*`
8			`* Authors:`
9			`* Jan-Bernd Themann <themann@de.ibm.com>`
10			`* Christoph Raisch <raisch@de.ibm.com>`
11			`*`
12			`*`
13			`* This program is free software; you can redistribute it and/or modify`
14			`* it under the terms of the GNU General Public License as published by`
15			`* the Free Software Foundation; either version 2, or (at your option)`
16			`* any later version.`
17			`*`
18			`* This program is distributed in the hope that it will be useful,`
19			`* but WITHOUT ANY WARRANTY; without even the implied warranty of`
20			`* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the`
21			`* GNU General Public License for more details.`
22			`*`
23			`* You should have received a copy of the GNU General Public License`
24			`* along with this program; if not, write to the Free Software`
25			`* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.`
26			`*/`
27
28
29			`#include <linux/module.h>`
30			`#include <linux/if_vlan.h>`
31			`#include <linux/inet_lro.h>`
32
33			`MODULE_LICENSE("GPL");`
34			`MODULE_AUTHOR("Jan-Bernd Themann <themann@de.ibm.com>");`
35			`MODULE_DESCRIPTION("Large Receive Offload (ipv4 / tcp)");`
36
37			`#define TCP_HDR_LEN(tcph) (tcph->doff << 2)`
38			`#define IP_HDR_LEN(iph) (iph->ihl << 2)`
39			`#define TCP_PAYLOAD_LENGTH(iph, tcph) \`
40			`(ntohs(iph->tot_len) - IP_HDR_LEN(iph) - TCP_HDR_LEN(tcph))`
41
42			`#define IPH_LEN_WO_OPTIONS 5`
43			`#define TCPH_LEN_WO_OPTIONS 5`
44			`#define TCPH_LEN_W_TIMESTAMP 8`
45
46			`#define LRO_MAX_PG_HLEN 64`
47
48			`#define LRO_INC_STATS(lro_mgr, attr) { lro_mgr->stats.attr++; }`
49
50			`/*`
51			`* Basic tcp checks whether packet is suitable for LRO`
52			`*/`
53
54			`static int lro_tcp_ip_check(struct iphdr iph, struct tcphdr tcph,`
55			`int len, struct net_lro_desc *lro_desc)`
56			`{`
57			`/* check ip header: don't aggregate padded frames */`
58			`if (ntohs(iph->tot_len) != len)`
59			`return -1;`
60
61			`if (TCP_PAYLOAD_LENGTH(iph, tcph) == 0)`
62			`return -1;`
63
64			`if (iph->ihl != IPH_LEN_WO_OPTIONS)`
65			`return -1;`
66
67			`if (tcph->cwr \|\| tcph->ece \|\| tcph->urg \|\| !tcph->ack`
68			`\|\| tcph->rst \|\| tcph->syn \|\| tcph->fin)`
69			`return -1;`
70
71			`if (INET_ECN_is_ce(ipv4_get_dsfield(iph)))`
72			`return -1;`
73
74			`if (tcph->doff != TCPH_LEN_WO_OPTIONS`
75			`&& tcph->doff != TCPH_LEN_W_TIMESTAMP)`
76			`return -1;`
77
78			`/* check tcp options (only timestamp allowed) */`
79			`if (tcph->doff == TCPH_LEN_W_TIMESTAMP) {`
80			`__be32 topt = (__be32 )(tcph + 1);`
81
82			`if (*topt != htonl((TCPOPT_NOP << 24) \| (TCPOPT_NOP << 16)`
83			`\| (TCPOPT_TIMESTAMP << 8)`
84			`\| TCPOLEN_TIMESTAMP))`
85			`return -1;`
86
87			`/* timestamp should be in right order */`
88			`topt++;`
89			`if (lro_desc && after(ntohl(lro_desc->tcp_rcv_tsval),`
90			`ntohl(*topt)))`
91			`return -1;`
92
93			`/* timestamp reply should not be zero */`
94			`topt++;`
95			`if (*topt == 0)`
96			`return -1;`
97			`}`
98
99			`return 0;`
100			`}`
101
102			`static void lro_update_tcp_ip_header(struct net_lro_desc *lro_desc)`
103			`{`
104			`struct iphdr *iph = lro_desc->iph;`
105			`struct tcphdr *tcph = lro_desc->tcph;`
106			`__be32 *p;`
107			`__wsum tcp_hdr_csum;`
108
109			`tcph->ack_seq = lro_desc->tcp_ack;`
110			`tcph->window = lro_desc->tcp_window;`
111
112			`if (lro_desc->tcp_saw_tstamp) {`
113			`p = (__be32 *)(tcph + 1);`
114			`*(p+2) = lro_desc->tcp_rcv_tsecr;`
115			`}`
116
117			`iph->tot_len = htons(lro_desc->ip_tot_len);`
118
119			`iph->check = 0;`
120			`iph->check = ip_fast_csum((u8 *)lro_desc->iph, iph->ihl);`
121
122			`tcph->check = 0;`
123			`tcp_hdr_csum = csum_partial((u8 *)tcph, TCP_HDR_LEN(tcph), 0);`
124			`lro_desc->data_csum = csum_add(lro_desc->data_csum, tcp_hdr_csum);`
125			`tcph->check = csum_tcpudp_magic(iph->saddr, iph->daddr,`
126			`lro_desc->ip_tot_len -`
127			`IP_HDR_LEN(iph), IPPROTO_TCP,`
128			`lro_desc->data_csum);`
129			`}`
130
131			`static __wsum lro_tcp_data_csum(struct iphdr iph, struct tcphdr tcph, int len)`
132			`{`
133			`__wsum tcp_csum;`
134			`__wsum tcp_hdr_csum;`
135			`__wsum tcp_ps_hdr_csum;`
136
137			`tcp_csum = ~csum_unfold(tcph->check);`
138			`tcp_hdr_csum = csum_partial((u8 *)tcph, TCP_HDR_LEN(tcph), tcp_csum);`
139
140			`tcp_ps_hdr_csum = csum_tcpudp_nofold(iph->saddr, iph->daddr,`
141			`len + TCP_HDR_LEN(tcph),`
142			`IPPROTO_TCP, 0);`
143
144			`return csum_sub(csum_sub(tcp_csum, tcp_hdr_csum),`
145			`tcp_ps_hdr_csum);`
146			`}`
147
148			`static void lro_init_desc(struct net_lro_desc lro_desc, struct sk_buff skb,`
149			`struct iphdr iph, struct tcphdr tcph,`
150			`u16 vlan_tag, struct vlan_group *vgrp)`
151			`{`
152			`int nr_frags;`
153			`__be32 *ptr;`
154			`u32 tcp_data_len = TCP_PAYLOAD_LENGTH(iph, tcph);`
155
156			`nr_frags = skb_shinfo(skb)->nr_frags;`
157			`lro_desc->parent = skb;`
158			`lro_desc->next_frag = &(skb_shinfo(skb)->frags[nr_frags]);`
159			`lro_desc->iph = iph;`
160			`lro_desc->tcph = tcph;`
161			`lro_desc->tcp_next_seq = ntohl(tcph->seq) + tcp_data_len;`
162			`lro_desc->tcp_ack = tcph->ack_seq;`
163			`lro_desc->tcp_window = tcph->window;`
164
165			`lro_desc->pkt_aggr_cnt = 1;`
166			`lro_desc->ip_tot_len = ntohs(iph->tot_len);`
167
168			`if (tcph->doff == 8) {`
169			`ptr = (__be32 *)(tcph+1);`
170			`lro_desc->tcp_saw_tstamp = 1;`
171			`lro_desc->tcp_rcv_tsval = *(ptr+1);`
172			`lro_desc->tcp_rcv_tsecr = *(ptr+2);`
173			`}`
174
175			`lro_desc->mss = tcp_data_len;`
176			`lro_desc->vgrp = vgrp;`
177			`lro_desc->vlan_tag = vlan_tag;`
178			`lro_desc->active = 1;`
179
180			`lro_desc->data_csum = lro_tcp_data_csum(iph, tcph,`
181			`tcp_data_len);`
182			`}`
183
184			`static inline void lro_clear_desc(struct net_lro_desc *lro_desc)`
185			`{`
186			`memset(lro_desc, 0, sizeof(struct net_lro_desc));`
187			`}`
188
189			`static void lro_add_common(struct net_lro_desc lro_desc, struct iphdr iph,`
190			`struct tcphdr *tcph, int tcp_data_len)`
191			`{`
192			`struct sk_buff *parent = lro_desc->parent;`
193			`__be32 *topt;`
194
195			`lro_desc->pkt_aggr_cnt++;`
196			`lro_desc->ip_tot_len += tcp_data_len;`
197			`lro_desc->tcp_next_seq += tcp_data_len;`
198			`lro_desc->tcp_window = tcph->window;`
199			`lro_desc->tcp_ack = tcph->ack_seq;`
200
201			`/* don't update tcp_rcv_tsval, would not work with PAWS */`
202			`if (lro_desc->tcp_saw_tstamp) {`
203			`topt = (__be32 *) (tcph + 1);`
204			`lro_desc->tcp_rcv_tsecr = *(topt + 2);`
205			`}`
206
207			`lro_desc->data_csum = csum_block_add(lro_desc->data_csum,`
208			`lro_tcp_data_csum(iph, tcph,`
209			`tcp_data_len),`
210			`parent->len);`
211
212			`parent->len += tcp_data_len;`
213			`parent->data_len += tcp_data_len;`
214			`if (tcp_data_len > lro_desc->mss)`
215			`lro_desc->mss = tcp_data_len;`
216			`}`
217
218			`static void lro_add_packet(struct net_lro_desc lro_desc, struct sk_buff skb,`
219			`struct iphdr iph, struct tcphdr tcph)`
220			`{`
221			`struct sk_buff *parent = lro_desc->parent;`
222			`int tcp_data_len = TCP_PAYLOAD_LENGTH(iph, tcph);`
223
224			`lro_add_common(lro_desc, iph, tcph, tcp_data_len);`
225
226			`skb_pull(skb, (skb->len - tcp_data_len));`
227			`parent->truesize += skb->truesize;`
228
229			`if (lro_desc->last_skb)`
230			`lro_desc->last_skb->next = skb;`
231			`else`
232			`skb_shinfo(parent)->frag_list = skb;`
233
234			`lro_desc->last_skb = skb;`
235			`}`
236
237			`static void lro_add_frags(struct net_lro_desc *lro_desc,`
238			`int len, int hlen, int truesize,`
239			`struct skb_frag_struct *skb_frags,`
240			`struct iphdr iph, struct tcphdr tcph)`
241			`{`
242			`struct sk_buff *skb = lro_desc->parent;`
243			`int tcp_data_len = TCP_PAYLOAD_LENGTH(iph, tcph);`
244
245			`lro_add_common(lro_desc, iph, tcph, tcp_data_len);`
246
247			`skb->truesize += truesize;`
248
249			`skb_frags[0].page_offset += hlen;`
250			`skb_frags[0].size -= hlen;`
251
252			`while (tcp_data_len > 0) {`
253			`(lro_desc->next_frag) = skb_frags;`
254			`tcp_data_len -= skb_frags->size;`
255			`lro_desc->next_frag++;`
256			`skb_frags++;`
257			`skb_shinfo(skb)->nr_frags++;`
258			`}`
259			`}`
260
261			`static int lro_check_tcp_conn(struct net_lro_desc *lro_desc,`
262			`struct iphdr *iph,`
263			`struct tcphdr *tcph)`
264			`{`
265			`if ((lro_desc->iph->saddr != iph->saddr)`
266			`\|\| (lro_desc->iph->daddr != iph->daddr)`
267			`\|\| (lro_desc->tcph->source != tcph->source)`
268			`\|\| (lro_desc->tcph->dest != tcph->dest))`
269			`return -1;`
270			`return 0;`
271			`}`
272
273			`static struct net_lro_desc lro_get_desc(struct net_lro_mgr lro_mgr,`
274			`struct net_lro_desc *lro_arr,`
275			`struct iphdr *iph,`
276			`struct tcphdr *tcph)`
277			`{`
278			`struct net_lro_desc *lro_desc = NULL;`
279			`struct net_lro_desc *tmp;`
280			`int max_desc = lro_mgr->max_desc;`
281			`int i;`
282
283			`for (i = 0; i < max_desc; i++) {`
284			`tmp = &lro_arr[i];`
285			`if (tmp->active)`
286			`if (!lro_check_tcp_conn(tmp, iph, tcph)) {`
287			`lro_desc = tmp;`
288			`goto out;`
289			`}`
290			`}`
291
292			`for (i = 0; i < max_desc; i++) {`
293			`if (!lro_arr[i].active) {`
294			`lro_desc = &lro_arr[i];`
295			`goto out;`
296			`}`
297			`}`
298
299			`LRO_INC_STATS(lro_mgr, no_desc);`
300			`out:`
301			`return lro_desc;`
302			`}`
303
304			`static void lro_flush(struct net_lro_mgr *lro_mgr,`
305			`struct net_lro_desc *lro_desc)`
306			`{`
307			`if (lro_desc->pkt_aggr_cnt > 1)`
308			`lro_update_tcp_ip_header(lro_desc);`
309
310			`skb_shinfo(lro_desc->parent)->gso_size = lro_desc->mss;`
311
312			`if (lro_desc->vgrp) {`
313			`if (lro_mgr->features & LRO_F_NAPI)`
314			`vlan_hwaccel_receive_skb(lro_desc->parent,`
315			`lro_desc->vgrp,`
316			`lro_desc->vlan_tag);`
317			`else`
318			`vlan_hwaccel_rx(lro_desc->parent,`
319			`lro_desc->vgrp,`
320			`lro_desc->vlan_tag);`
321
322			`} else {`
323			`if (lro_mgr->features & LRO_F_NAPI)`
324			`netif_receive_skb(lro_desc->parent);`
325			`else`
326			`netif_rx(lro_desc->parent);`
327			`}`
328
329			`LRO_INC_STATS(lro_mgr, flushed);`
330			`lro_clear_desc(lro_desc);`
331			`}`
332
333			`static int __lro_proc_skb(struct net_lro_mgr lro_mgr, struct sk_buff skb,`
334			`struct vlan_group vgrp, u16 vlan_tag, void priv)`
335			`{`
336			`struct net_lro_desc *lro_desc;`
337			`struct iphdr *iph;`
338			`struct tcphdr *tcph;`
339			`u64 flags;`
340			`int vlan_hdr_len = 0;`
341
342			`if (!lro_mgr->get_skb_header`
343			`\|\| lro_mgr->get_skb_header(skb, (void )&iph, (void )&tcph,`
344			`&flags, priv))`
345			`goto out;`
346
347			`if (!(flags & LRO_IPV4) \|\| !(flags & LRO_TCP))`
348			`goto out;`
349
350			`lro_desc = lro_get_desc(lro_mgr, lro_mgr->lro_arr, iph, tcph);`
351			`if (!lro_desc)`
352			`goto out;`
353
354			`if ((skb->protocol == htons(ETH_P_8021Q))`
355			`&& !(lro_mgr->features & LRO_F_EXTRACT_VLAN_ID))`
356			`vlan_hdr_len = VLAN_HLEN;`
357
358			`if (!lro_desc->active) { /* start new lro session */`
359			`if (lro_tcp_ip_check(iph, tcph, skb->len - vlan_hdr_len, NULL))`
360			`goto out;`
361
362			`skb->ip_summed = lro_mgr->ip_summed_aggr;`
363			`lro_init_desc(lro_desc, skb, iph, tcph, vlan_tag, vgrp);`
364			`LRO_INC_STATS(lro_mgr, aggregated);`
365			`return 0;`
366			`}`
367
368			`if (lro_desc->tcp_next_seq != ntohl(tcph->seq))`
369			`goto out2;`
370
371			`if (lro_tcp_ip_check(iph, tcph, skb->len, lro_desc))`
372			`goto out2;`
373
374			`lro_add_packet(lro_desc, skb, iph, tcph);`
375			`LRO_INC_STATS(lro_mgr, aggregated);`
376
377			`if ((lro_desc->pkt_aggr_cnt >= lro_mgr->max_aggr) \|\|`
378			`lro_desc->parent->len > (0xFFFF - lro_mgr->dev->mtu))`
379			`lro_flush(lro_mgr, lro_desc);`
380
381			`return 0;`
382
383			`out2: /* send aggregated SKBs to stack */`
384			`lro_flush(lro_mgr, lro_desc);`
385
386			`out: /* Original SKB has to be posted to stack */`
387			`skb->ip_summed = lro_mgr->ip_summed;`
388			`return 1;`
389			`}`
390
391
392			`static struct sk_buff lro_gen_skb(struct net_lro_mgr lro_mgr,`
393			`struct skb_frag_struct *frags,`
394			`int len, int true_size,`
395			`void *mac_hdr,`
396			`int hlen, __wsum sum,`
397			`u32 ip_summed)`
398			`{`
399			`struct sk_buff *skb;`
400			`struct skb_frag_struct *skb_frags;`
401			`int data_len = len;`
402			`int hdr_len = min(len, hlen);`
403
404			`skb = netdev_alloc_skb(lro_mgr->dev, hlen + lro_mgr->frag_align_pad);`
405			`if (!skb)`
406			`return NULL;`
407
408			`skb_reserve(skb, lro_mgr->frag_align_pad);`
409			`skb->len = len;`
410			`skb->data_len = len - hdr_len;`
411			`skb->truesize += true_size;`
412			`skb->tail += hdr_len;`
413
414			`memcpy(skb->data, mac_hdr, hdr_len);`
415
416			`skb_frags = skb_shinfo(skb)->frags;`
417			`while (data_len > 0) {`
418			`skb_frags = frags;`
419			`data_len -= frags->size;`
420			`skb_frags++;`
421			`frags++;`
422			`skb_shinfo(skb)->nr_frags++;`
423			`}`
424
425			`skb_shinfo(skb)->frags[0].page_offset += hdr_len;`
426			`skb_shinfo(skb)->frags[0].size -= hdr_len;`
427
428			`skb->ip_summed = ip_summed;`
429			`skb->csum = sum;`
430			`skb->protocol = eth_type_trans(skb, lro_mgr->dev);`
431			`return skb;`
432			`}`
433
434			`static struct sk_buff __lro_proc_segment(struct net_lro_mgr lro_mgr,`
435			`struct skb_frag_struct *frags,`
436			`int len, int true_size,`
437			`struct vlan_group *vgrp,`
438			`u16 vlan_tag, void *priv, __wsum sum)`
439			`{`
440			`struct net_lro_desc *lro_desc;`
441			`struct iphdr *iph;`
442			`struct tcphdr *tcph;`
443			`struct sk_buff *skb;`
444			`u64 flags;`
445			`void *mac_hdr;`
446			`int mac_hdr_len;`
447			`int hdr_len = LRO_MAX_PG_HLEN;`
448			`int vlan_hdr_len = 0;`
449
450			`if (!lro_mgr->get_frag_header`
451			`\|\| lro_mgr->get_frag_header(frags, (void )&mac_hdr, (void )&iph,`
452			`(void *)&tcph, &flags, priv)) {`
453			`mac_hdr = page_address(frags->page) + frags->page_offset;`
454			`goto out1;`
455			`}`
456
457			`if (!(flags & LRO_IPV4) \|\| !(flags & LRO_TCP))`
458			`goto out1;`
459
460			`hdr_len = (int)((void *)(tcph) + TCP_HDR_LEN(tcph) - mac_hdr);`
461			`mac_hdr_len = (int)((void *)(iph) - mac_hdr);`
462
463			`lro_desc = lro_get_desc(lro_mgr, lro_mgr->lro_arr, iph, tcph);`
464			`if (!lro_desc)`
465			`goto out1;`
466
467			`if (!lro_desc->active) { /* start new lro session */`
468			`if (lro_tcp_ip_check(iph, tcph, len - mac_hdr_len, NULL))`
469			`goto out1;`
470
471			`skb = lro_gen_skb(lro_mgr, frags, len, true_size, mac_hdr,`
472			`hdr_len, 0, lro_mgr->ip_summed_aggr);`
473			`if (!skb)`
474			`goto out;`
475
476			`if ((skb->protocol == htons(ETH_P_8021Q))`
477			`&& !(lro_mgr->features & LRO_F_EXTRACT_VLAN_ID))`
478			`vlan_hdr_len = VLAN_HLEN;`
479
480			`iph = (void *)(skb->data + vlan_hdr_len);`
481			`tcph = (void )((u8 )skb->data + vlan_hdr_len`
482			`+ IP_HDR_LEN(iph));`
483
484			`lro_init_desc(lro_desc, skb, iph, tcph, 0, NULL);`
485			`LRO_INC_STATS(lro_mgr, aggregated);`
486			`return NULL;`
487			`}`
488
489			`if (lro_desc->tcp_next_seq != ntohl(tcph->seq))`
490			`goto out2;`
491
492			`if (lro_tcp_ip_check(iph, tcph, len - mac_hdr_len, lro_desc))`
493			`goto out2;`
494
495			`lro_add_frags(lro_desc, len, hdr_len, true_size, frags, iph, tcph);`
496			`LRO_INC_STATS(lro_mgr, aggregated);`
497
498			`if ((skb_shinfo(lro_desc->parent)->nr_frags >= lro_mgr->max_aggr) \|\|`
499			`lro_desc->parent->len > (0xFFFF - lro_mgr->dev->mtu))`
500			`lro_flush(lro_mgr, lro_desc);`
501
502			`return NULL;`
503
504			`out2: /* send aggregated packets to the stack */`
505			`lro_flush(lro_mgr, lro_desc);`
506
507			`out1: /* Original packet has to be posted to the stack */`
508			`skb = lro_gen_skb(lro_mgr, frags, len, true_size, mac_hdr,`
509			`hdr_len, sum, lro_mgr->ip_summed);`
510			`out:`
511			`return skb;`
512			`}`
513
514			`void lro_receive_skb(struct net_lro_mgr *lro_mgr,`
515			`struct sk_buff *skb,`
516			`void *priv)`
517			`{`
518			`if (__lro_proc_skb(lro_mgr, skb, NULL, 0, priv)) {`
519			`if (lro_mgr->features & LRO_F_NAPI)`
520			`netif_receive_skb(skb);`
521			`else`
522			`netif_rx(skb);`
523			`}`
524			`}`
525			`EXPORT_SYMBOL(lro_receive_skb);`
526
527			`void lro_vlan_hwaccel_receive_skb(struct net_lro_mgr *lro_mgr,`
528			`struct sk_buff *skb,`
529			`struct vlan_group *vgrp,`
530			`u16 vlan_tag,`
531			`void *priv)`
532			`{`
533			`if (__lro_proc_skb(lro_mgr, skb, vgrp, vlan_tag, priv)) {`
534			`if (lro_mgr->features & LRO_F_NAPI)`
535			`vlan_hwaccel_receive_skb(skb, vgrp, vlan_tag);`
536			`else`
537			`vlan_hwaccel_rx(skb, vgrp, vlan_tag);`
538			`}`
539			`}`
540			`EXPORT_SYMBOL(lro_vlan_hwaccel_receive_skb);`
541
542			`void lro_receive_frags(struct net_lro_mgr *lro_mgr,`
543			`struct skb_frag_struct *frags,`
544			`int len, int true_size, void *priv, __wsum sum)`
545			`{`
546			`struct sk_buff *skb;`
547
548			`skb = __lro_proc_segment(lro_mgr, frags, len, true_size, NULL, 0,`
549			`priv, sum);`
550			`if (!skb)`
551			`return;`
552
553			`if (lro_mgr->features & LRO_F_NAPI)`
554			`netif_receive_skb(skb);`
555			`else`
556			`netif_rx(skb);`
557			`}`
558			`EXPORT_SYMBOL(lro_receive_frags);`
559
560			`void lro_vlan_hwaccel_receive_frags(struct net_lro_mgr *lro_mgr,`
561			`struct skb_frag_struct *frags,`
562			`int len, int true_size,`
563			`struct vlan_group *vgrp,`
564			`u16 vlan_tag, void *priv, __wsum sum)`
565			`{`
566			`struct sk_buff *skb;`
567
568			`skb = __lro_proc_segment(lro_mgr, frags, len, true_size, vgrp,`
569			`vlan_tag, priv, sum);`
570			`if (!skb)`
571			`return;`
572
573			`if (lro_mgr->features & LRO_F_NAPI)`
574			`vlan_hwaccel_receive_skb(skb, vgrp, vlan_tag);`
575			`else`
576			`vlan_hwaccel_rx(skb, vgrp, vlan_tag);`
577			`}`
578			`EXPORT_SYMBOL(lro_vlan_hwaccel_receive_frags);`
579
580			`void lro_flush_all(struct net_lro_mgr *lro_mgr)`
581			`{`
582			`int i;`
583			`struct net_lro_desc *lro_desc = lro_mgr->lro_arr;`
584
585			`for (i = 0; i < lro_mgr->max_desc; i++) {`
586			`if (lro_desc[i].active)`
587			`lro_flush(lro_mgr, &lro_desc[i]);`
588			`}`
589			`}`
590			`EXPORT_SYMBOL(lro_flush_all);`
591
592			`void lro_flush_pkt(struct net_lro_mgr *lro_mgr,`
593			`struct iphdr iph, struct tcphdr tcph)`
594			`{`
595			`struct net_lro_desc *lro_desc;`
596
597			`lro_desc = lro_get_desc(lro_mgr, lro_mgr->lro_arr, iph, tcph);`
598			`if (lro_desc->active)`
599			`lro_flush(lro_mgr, lro_desc);`
600			`}`
601			`EXPORT_SYMBOL(lro_flush_pkt);`