From: Rusty Russell <rusty@rustcorp.com.au>
Subject: virtio: expose added descriptors immediately.

A virtio driver does virtqueue_add_buf() multiple times before finally
calling virtqueue_kick(); previously we only exposed the added buffers
in the virtqueue_kick() call.  This means we don't need a memory
barrier in virtqueue_add_buf(), but it reduces concurrency as the
device (ie. host) can't see the buffers until the kick.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
---
 drivers/virtio/virtio_ring.c |   37 ++++++++++++++++++++++---------------
 1 file changed, 22 insertions(+), 15 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -227,8 +227,13 @@ add_head:
 
 	/* Put entry in available array (but don't update avail->idx until they
 	 * do sync).  FIXME: avoid modulus here? */
-	avail = (vq->vring.avail->idx + vq->num_added++) % vq->vring.num;
+	avail = vq->vring.avail->idx % vq->vring.num;
 	vq->vring.avail->ring[avail] = head;
+	/* Descriptors and available array need to be set before we expose the
+	 * new available array entries. */
+	virtio_wmb();
+	vq->vring.avail->idx++;
+	vq->num_added++;
 
 	pr_debug("Added buffer head %i to %p\n", head, vq);
 	END_USE(vq);
@@ -237,28 +242,30 @@ add_head:
 }
 EXPORT_SYMBOL_GPL(virtqueue_add_buf_gfp);
 
+static bool need_event(const struct vring_virtqueue *vq)
+{
+	/* Need to update avail index before checking if we should notify */
+	virtio_mb();
+
+	if (vq->event) {
+		u16 old = vq->vring.avail->idx - vq->num_added;
+		return vring_need_event(vring_avail_event(&vq->vring),
+					vq->vring.avail->idx, old);
+	}
+	return vq->vring.used->flags & VRING_USED_F_NO_NOTIFY;
+}
+
 void virtqueue_kick(struct virtqueue *_vq)
 {
 	struct vring_virtqueue *vq = to_vvq(_vq);
-	u16 new, old;
 	START_USE(vq);
-	/* Descriptors and available array need to be set before we expose the
-	 * new available array entries. */
-	virtio_wmb();
 
-	old = vq->vring.avail->idx;
-	new = vq->vring.avail->idx = old + vq->num_added;
-	vq->num_added = 0;
-
-	/* Need to update avail index before checking if we should notify */
-	virtio_mb();
-
-	if (vq->event ?
-	    vring_need_event(vring_avail_event(&vq->vring), new, old) :
-	    !(vq->vring.used->flags & VRING_USED_F_NO_NOTIFY))
+	if (need_event(vq)) {
 		/* Prod other side to tell it about changes. */
 		vq->notify(&vq->vq);
+	}
 
+	vq->num_added = 0;
 	END_USE(vq);
 }
 EXPORT_SYMBOL_GPL(virtqueue_kick);