This implements partial checksum write support for tun/tap. We use the virtio_net_hdr: it is an ABI already and designed to encapsulate such metadata as GSO (in coming patches) and partial checksums. Performance before: --- drivers/net/tun.c | 202 +++++++++++++++++++++++++++++++++++++++++------ drivers/net/virtio_net.c | 2 include/linux/if_tun.h | 2 3 files changed, 181 insertions(+), 25 deletions(-) diff -r 7db85002cc09 drivers/net/tun.c --- a/drivers/net/tun.c Thu Jan 10 16:52:00 2008 +1100 +++ b/drivers/net/tun.c Thu Jan 10 17:44:33 2008 +1100 @@ -62,6 +62,7 @@ #include #include #include +#include #include #include @@ -238,11 +239,162 @@ static unsigned int tun_chr_poll(struct return mask; } +static struct sk_buff *copy_user_skb(size_t align, struct iovec *iv, size_t len) +{ + struct sk_buff *skb; + + if (!(skb = alloc_skb(len + align, GFP_KERNEL))) + return ERR_PTR(-ENOMEM); + + if (align) + skb_reserve(skb, align); + + if (memcpy_fromiovec(skb_put(skb, len), iv, len)) { + kfree_skb(skb); + return ERR_PTR(-EFAULT); + } + return skb; +} + +/* This will fail if they give us a crazy iovec, but that's their own fault. */ +static int get_user_skb_frags(const struct iovec *iv, size_t count, + struct skb_frag_struct *f) +{ + unsigned int i, j, num_pg = 0; + int err; + struct page *pages[MAX_SKB_FRAGS]; + + down_read(¤t->mm->mmap_sem); + for (i = 0; i < count; i++) { + int expect, n; + unsigned long base, len; + base = (unsigned long)iv[count].iov_base; + len = (unsigned long)iv[count].iov_len; + + /* How many pages will this take? */ + expect = (base + len)/PAGE_SIZE - base/PAGE_SIZE; + if (unlikely(num_pg + expect > MAX_SKB_FRAGS)) { + err = -ENOSPC; + goto fail; + } + n = get_user_pages(current, current->mm, base, len, + 0, 0, pages, NULL); + if (unlikely(n < 0)) { + err = n; + goto fail; + } + + /* Transfer pages to the frag array */ + for (j = 0; j < n; j++) { + f[num_pg].page = pages[j]; + if (j == 0) { + f[num_pg].page_offset = offset_in_page(base); + f[num_pg].size = min(len, PAGE_SIZE - + f[num_pg].page_offset); + } else { + f[num_pg].page_offset = 0; + f[num_pg].size = min(len, PAGE_SIZE); + } + len -= f[num_pg].size; + base += f[num_pg].size; + num_pg++; + } + + if (unlikely(n != expect)) { + err = -EFAULT; + goto fail; + } + } + up_read(¤t->mm->mmap_sem); + return num_pg; + +fail: + for (i = 0; i < num_pg; i++) + put_page(f[i].page); + up_read(¤t->mm->mmap_sem); + return err; +} + + +static struct sk_buff *map_user_skb(const struct virtio_net_hdr *gso, + size_t align, struct iovec *iv, + size_t count, size_t len) +{ + struct sk_buff *skb; + struct skb_shared_info *sinfo; + int err; + + if (gso->gso_hdr_len > len) + return ERR_PTR(-EINVAL); + + if (!(skb = alloc_skb(gso->gso_hdr_len + align, GFP_KERNEL))) + return ERR_PTR(-ENOMEM); + + if (align) + skb_reserve(skb, align); + + sinfo = skb_shinfo(skb); + sinfo->gso_size = gso->gso_size; + sinfo->gso_type = SKB_GSO_DODGY; + switch (gso->gso_type) { + case VIRTIO_NET_HDR_GSO_TCPV4_ECN: + sinfo->gso_type |= SKB_GSO_TCP_ECN; + /* fall through */ + case VIRTIO_NET_HDR_GSO_TCPV4: + sinfo->gso_type |= SKB_GSO_TCPV4; + break; + case VIRTIO_NET_HDR_GSO_TCPV6: + sinfo->gso_type |= SKB_GSO_TCPV6; + break; + case VIRTIO_NET_HDR_GSO_UDP: + sinfo->gso_type |= SKB_GSO_UDP; + break; + default: + err = -EINVAL; + goto fail; + } + + /* Copy in the header. */ + if (memcpy_fromiovec(skb_put(skb, gso->gso_hdr_len), iv, + gso->gso_hdr_len)) { + err = -EFAULT; + goto fail; + } + + err = get_user_skb_frags(iv, count, sinfo->frags); + if (err < 0) + goto fail; + + sinfo->nr_frags = err; + + /* GSO code expects transport header set up */ + skb_set_transport_header(skb, gso->gso_hdr_len); + + return skb; + +fail: + kfree_skb(skb); + return ERR_PTR(err); +} + +static inline size_t iov_total(const struct iovec *iv, unsigned long count) +{ + unsigned long i; + size_t len; + + for (i = 0, len = 0; i < count; i++) + len += iv[i].iov_len; + + return len; +} + /* Get packet from user space buffer */ -static __inline__ ssize_t tun_get_user(struct tun_struct *tun, struct iovec *iv, size_t count) +static __inline__ ssize_t tun_get_user(struct tun_struct *tun, struct iovec *iv, size_t num) { struct tun_pi pi = { 0, __constant_htons(ETH_P_IP) }; + struct virtio_net_hdr gso = { 0, VIRTIO_NET_HDR_GSO_NONE }; struct sk_buff *skb; + size_t count = iov_total(iv, num); size_t len = count, align = 0; if (!(tun->flags & TUN_NO_PI)) { @@ -252,21 +404,25 @@ static __inline__ ssize_t tun_get_user(s if(memcpy_fromiovec((void *)&pi, iv, sizeof(pi))) return -EFAULT; } + if (tun->flags & TUN_GSO_HDR) { + if ((len -= sizeof(gso)) > count) + return -EINVAL; + + if (memcpy_fromiovec((void *)&gso, iv, sizeof(gso))) + return -EFAULT; + } if ((tun->flags & TUN_TYPE_MASK) == TUN_TAP_DEV) align = NET_IP_ALIGN; - if (!(skb = alloc_skb(len + align, GFP_KERNEL))) { + if (gso.gso_type != VIRTIO_NET_HDR_GSO_NONE) + skb = map_user_skb(&gso, align, iv, count, len); + else + skb = copy_user_skb(align, iv, len); + + if (IS_ERR(skb)) { tun->dev->stats.rx_dropped++; - return -ENOMEM; - } - - if (align) - skb_reserve(skb, align); - if (memcpy_fromiovec(skb_put(skb, len), iv, len)) { - tun->dev->stats.rx_dropped++; - kfree_skb(skb); - return -EFAULT; + return PTR_ERR(skb); } switch (tun->flags & TUN_TYPE_MASK) { @@ -280,7 +436,13 @@ static __inline__ ssize_t tun_get_user(s break; }; - if (tun->flags & TUN_NOCHECKSUM) + if (gso.flags & (1 << VIRTIO_NET_F_NO_CSUM)) { + if (!skb_partial_csum_set(skb,gso.csum_start,gso.csum_offset)) { + tun->dev->stats.rx_dropped++; + kfree_skb(skb); + return -EINVAL; + } + } else if (tun->flags & TUN_NOCHECKSUM) skb->ip_summed = CHECKSUM_UNNECESSARY; netif_rx_ni(skb); @@ -290,17 +452,6 @@ static __inline__ ssize_t tun_get_user(s tun->dev->stats.rx_bytes += len; return count; -} - -static inline size_t iov_total(const struct iovec *iv, unsigned long count) -{ - unsigned long i; - size_t len; - - for (i = 0, len = 0; i < count; i++) - len += iv[i].iov_len; - - return len; } static ssize_t tun_chr_aio_write(struct kiocb *iocb, const struct iovec *iv, @@ -313,7 +464,7 @@ static ssize_t tun_chr_aio_write(struct DBG(KERN_INFO "%s: tun_chr_write %ld\n", tun->dev->name, count); - return tun_get_user(tun, (struct iovec *) iv, iov_total(iv, count)); + return tun_get_user(tun, (struct iovec *) iv, count); } /* Put packet to the user space buffer */ @@ -543,6 +694,9 @@ static int tun_set_iff(struct file *file if (ifr->ifr_flags & IFF_ONE_QUEUE) tun->flags |= TUN_ONE_QUEUE; + + if (ifr->ifr_flags & IFF_GSO_HDR) + tun->flags |= TUN_GSO_HDR; file->private_data = tun; tun->attached = 1; diff -r 7db85002cc09 drivers/net/virtio_net.c --- a/drivers/net/virtio_net.c Thu Jan 10 16:52:00 2008 +1100 +++ b/drivers/net/virtio_net.c Thu Jan 10 17:44:33 2008 +1100 @@ -126,7 +126,7 @@ static void receive_skb(struct net_devic /* Header must be checked, and gso_segs computed. */ skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY; skb_shinfo(skb)->gso_segs = 0; - skb_set_transport_header(skb, gso->gso_hdr_len); + skb_set_transport_header(skb, hdr->gso_hdr_len); } netif_receive_skb(skb); diff -r 7db85002cc09 include/linux/if_tun.h --- a/include/linux/if_tun.h Thu Jan 10 16:52:00 2008 +1100 +++ b/include/linux/if_tun.h Thu Jan 10 17:44:33 2008 +1100 @@ -70,6 +70,7 @@ struct tun_struct { #define TUN_NO_PI 0x0040 #define TUN_ONE_QUEUE 0x0080 #define TUN_PERSIST 0x0100 +#define TUN_GSO_HDR 0x0200 /* Ioctl defines */ #define TUNSETNOCSUM _IOW('T', 200, int) @@ -85,6 +86,7 @@ struct tun_struct { #define IFF_TAP 0x0002 #define IFF_NO_PI 0x1000 #define IFF_ONE_QUEUE 0x2000 +#define IFF_GSO_HDR 0x4000 struct tun_pi { unsigned short flags;