Experiment in making virtio_net use alloc_pskb Before: Netcat 1GB guest->host: 8.28 seconds netperf guest->host: 4277.53 Mb/sec After: Netcat 1GB guest->host: 8.57 seconds netperf guest->host: 3987.69 Mb/sec Signed-off-by: Rusty Russell --- drivers/net/virtio_net.c | 80 +++++++---------------------------------------- net/core/skbuff.c | 11 +++++- 2 files changed, 23 insertions(+), 68 deletions(-) diff -r 8e433bd4addb drivers/net/virtio_net.c --- a/drivers/net/virtio_net.c Mon Oct 20 20:40:43 2008 +1100 +++ b/drivers/net/virtio_net.c Mon Oct 20 21:01:38 2008 +1100 @@ -61,9 +61,6 @@ struct virtnet_info /* Receive & send queues. */ struct sk_buff_head recv; struct sk_buff_head send; - - /* Chain pages by the private ptr. */ - struct page *pages; }; static inline struct virtio_net_hdr *skb_vnet_hdr(struct sk_buff *skb) @@ -74,33 +71,6 @@ static inline void vnet_hdr_to_sg(struct static inline void vnet_hdr_to_sg(struct scatterlist *sg, struct sk_buff *skb) { sg_init_one(sg, skb_vnet_hdr(skb), sizeof(struct virtio_net_hdr)); -} - -static void give_a_page(struct virtnet_info *vi, struct page *page) -{ - page->private = (unsigned long)vi->pages; - vi->pages = page; -} - -static void trim_pages(struct virtnet_info *vi, struct sk_buff *skb) -{ - unsigned int i; - - for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) - give_a_page(vi, skb_shinfo(skb)->frags[i].page); - skb_shinfo(skb)->nr_frags = 0; - skb->data_len = 0; -} - -static struct page *get_a_page(struct virtnet_info *vi, gfp_t gfp_mask) -{ - struct page *p = vi->pages; - - if (p) - vi->pages = (struct page *)p->private; - else - p = alloc_page(gfp_mask); - return p; } static void skb_xmit_done(struct virtqueue *svq) @@ -122,7 +92,6 @@ static void receive_skb(struct net_devic unsigned len) { struct virtio_net_hdr *hdr = skb_vnet_hdr(skb); - int err; if (unlikely(len < sizeof(struct virtio_net_hdr) + ETH_HLEN)) { pr_debug("%s: short packet %i\n", dev->name, len); @@ -131,16 +100,7 @@ static void receive_skb(struct net_devic } len -= sizeof(struct virtio_net_hdr); - if (len <= MAX_PACKET_LEN) - trim_pages(dev->priv, skb); - - err = pskb_trim(skb, len); - if (err) { - pr_debug("%s: pskb_trim failed %i %d\n", dev->name, len, err); - dev->stats.rx_dropped++; - goto drop; - } - skb->truesize += skb->data_len; + trim_alloced_pskb(skb, len); dev->stats.rx_bytes += skb->len; dev->stats.rx_packets++; @@ -202,41 +162,31 @@ static void try_fill_recv(struct virtnet { struct sk_buff *skb; struct scatterlist sg[2+MAX_SKB_FRAGS]; - int num, err, i; + int num, err; sg_init_table(sg, 2+MAX_SKB_FRAGS); for (;;) { - skb = netdev_alloc_skb(vi->dev, MAX_PACKET_LEN); + if (vi->big_packets) { + skb = alloc_pskb(MAX_PACKET_LEN, + MAX_SKB_FRAGS*PAGE_SIZE, + GFP_ATOMIC); + } else { + skb = netdev_alloc_skb(vi->dev, MAX_PACKET_LEN); + if (skb) + skb_put(skb, MAX_PACKET_LEN); + } + if (unlikely(!skb)) break; - skb_put(skb, MAX_PACKET_LEN); vnet_hdr_to_sg(sg, skb); - - if (vi->big_packets) { - for (i = 0; i < MAX_SKB_FRAGS; i++) { - skb_frag_t *f = &skb_shinfo(skb)->frags[i]; - f->page = get_a_page(vi, GFP_ATOMIC); - if (!f->page) - break; - - f->page_offset = 0; - f->size = PAGE_SIZE; - - skb->data_len += PAGE_SIZE; - skb->len += PAGE_SIZE; - - skb_shinfo(skb)->nr_frags++; - } - } - num = skb_to_sgvec(skb, sg+1, 0, skb->len) + 1; skb_queue_head(&vi->recv, skb); err = vi->rvq->vq_ops->add_buf(vi->rvq, sg, 0, num, skb); if (err) { skb_unlink(skb, &vi->recv); - trim_pages(vi, skb); + trim_alloced_pskb(vi, skb_headlen(skb)); kfree_skb(skb); break; } @@ -540,7 +490,6 @@ static int virtnet_probe(struct virtio_d vi->dev = dev; vi->vdev = vdev; vdev->priv = vi; - vi->pages = NULL; /* If they give us a callback when all buffers are done, we don't need * the timer. */ @@ -627,9 +576,6 @@ static void virtnet_remove(struct virtio vdev->config->del_vq(vi->rvq); unregister_netdev(vi->dev); - while (vi->pages) - __free_pages(get_a_page(vi, GFP_KERNEL), 0); - free_netdev(vi->dev); } diff -r 8e433bd4addb net/core/skbuff.c --- a/net/core/skbuff.c Mon Oct 20 20:40:43 2008 +1100 +++ b/net/core/skbuff.c Mon Oct 20 21:01:38 2008 +1100 @@ -239,6 +239,11 @@ nodata: goto out; } +static unsigned int pages_given, pages_got, pages_alloced; +module_param(pages_given, uint, 0444); +module_param(pages_got, uint, 0444); +module_param(pages_alloced, uint, 0444); + static void give_pskb_page(struct page *p) { unsigned long flags; @@ -247,6 +252,7 @@ static void give_pskb_page(struct page * p->private = (unsigned long)pskb_pages_cache; pskb_pages_cache = p; pskb_pages_count++; + pages_given++; spin_unlock_irqrestore(&pskb_pages_lock, flags); } @@ -260,11 +266,14 @@ static struct page *get_pskb_page(gfp_t if (p) { pskb_pages_cache = (struct page *)(p->private); pskb_pages_count--; + pages_got++; } spin_unlock_irqrestore(&pskb_pages_lock, flags); - if (!p) + if (!p) { p = alloc_page(priority); + pages_alloced++; + } return p; }