Subject: vdevice tool for manipulating vdevice bus

This creates a simple C tool for list, adding and removing vdevices.
This is a demonstration (although, in my opinion, and important one,
because it shows how simple this is).  

This functionality should also be added to the python tools.

Index: xen-sane/.hgignore
===================================================================
--- xen-sane.orig/.hgignore	2006-06-28 14:36:41.000000000 +1000
+++ xen-sane/.hgignore	2006-07-03 14:07:32.000000000 +1000
@@ -146,6 +146,7 @@
 ^tools/security/secpol_tool$
 ^tools/security/xen/.*$
 ^tools/tests/test_x86_emulator$
+^tools/vdevice/vdevice$
 ^tools/vnet/gc$
 ^tools/vnet/gc.*/.*$
 ^tools/vnet/vnet-module/.*\.ko$
Index: xen-sane/tools/Makefile
===================================================================
--- xen-sane.orig/tools/Makefile	2006-06-28 14:36:41.000000000 +1000
+++ xen-sane/tools/Makefile	2006-07-03 14:07:32.000000000 +1000
@@ -16,6 +16,7 @@
 SUBDIRS-$(VTPM_TOOLS) += vtpm_manager
 SUBDIRS-$(VTPM_TOOLS) += vtpm
 SUBDIRS-y += xenstat
+SUBDIRS-y += vdevice
 
 # These don't cross-compile
 ifeq ($(XEN_COMPILE_ARCH),$(XEN_TARGET_ARCH))
Index: xen-sane/tools/vdevice/Makefile
===================================================================
--- /dev/null	1970-01-01 00:00:00.000000000 +0000
+++ xen-sane/tools/vdevice/Makefile	2006-07-03 14:07:32.000000000 +1000
@@ -0,0 +1,34 @@
+XEN_ROOT=../..
+include $(XEN_ROOT)/tools/Rules.mk
+
+INSTALL         = install
+INSTALL_DATA	= $(INSTALL) -m0644
+INSTALL_PROG    = $(INSTALL) -m0755
+INSTALL_DIR     = $(INSTALL) -d -m0755
+
+PROFILE=#-pg
+BASECFLAGS=-Wall -g -Werror
+# Make gcc generate dependencies.
+BASECFLAGS += -Wp,-MD,.$(@F).d
+PROG_DEP = .*.d
+BASECFLAGS+= -O3 $(PROFILE)
+BASECFLAGS+= -I$(XEN_ROOT)/tools/libxc
+BASECFLAGS+= -I$(XEN_ROOT)/tools/xenstore
+BASECFLAGS+= -I.
+
+CFLAGS  += $(BASECFLAGS)
+LDFLAGS += $(PROFILE) -L$(XEN_LIBXC) -L$(XEN_XENSTORE)
+
+all: vdevice
+
+clean:
+	rm -f vdevice *.o .*.d
+
+vdevice: vdevice.o
+	$(LINK.o) $^ $(LOADLIBES) $(LDLIBS) -lxenctrl -lxenstore -o $@
+
+install: vdevice
+	$(INSTALL_DIR) -p $(DESTDIR)/usr/sbin
+	$(INSTALL_PROG) vdevice $(DESTDIR)/usr/sbin
+
+-include $(PROG_DEP)
Index: xen-sane/tools/vdevice/vdevice.c
===================================================================
--- /dev/null	1970-01-01 00:00:00.000000000 +0000
+++ xen-sane/tools/vdevice/vdevice.c	2006-07-03 14:29:24.000000000 +1000
@@ -0,0 +1,571 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <err.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <assert.h>
+#include <errno.h>
+#include <stdint.h>
+#include <sys/ioctl.h>
+#include <net/ethernet.h>
+
+#include <xs.h>
+
+#include <xen/xen.h>
+#include <xen/share.h>
+#include <xen/linux/xenshare.h>
+#include <xen/event_channel.h>
+#include <xen/linux/privcmd.h>
+#include <xen/io/vdevice.h>
+
+#include <xc_private.h>
+
+#define PROGRAM_NAME "vdevice"
+
+static int xc_fd;
+
+#define __unused __attribute__((unused))
+
+/* FIXME: Move to xenctrl library */
+static int HYPERVISOR_share(int cmd, int arg1, int arg2, int arg3, int arg4)
+{
+	privcmd_hypercall_t privcmd;
+
+	privcmd.op = __HYPERVISOR_share_op;
+	privcmd.arg[0] = cmd;
+	privcmd.arg[1] = arg1;
+	privcmd.arg[2] = arg2;
+	privcmd.arg[3] = arg3;
+	privcmd.arg[4] = arg4;
+
+	return do_xen_hypercall(xc_fd, &privcmd);
+}
+
+/* FIXME: Move to xenctrl library */
+static int add_grant(share_ref_t ref, domid_t dom)
+{
+	dom0_op_t op = { .cmd = DOM0_GRANTSHAREDPAGES,
+		 .interface_version = DOM0_INTERFACE_VERSION,
+		 .u.grantsharedpages.share_ref = ref,
+		 .u.grantsharedpages.domain = dom };
+
+	/* FIXME: Skip domain 0 as it will always have access */
+	if (dom == 0)
+		return 0;
+
+	return do_dom0_op(xc_fd, &op);
+}
+
+static void *map_pages(share_ref_t share_ref, unsigned int num_pages,
+                       unsigned int *peer_id)
+{
+	struct xenshare_get_share shareget;
+	int shareiofd, ret;
+	void *sharepage;
+
+	shareiofd = open("/dev/xenshare", O_RDWR);
+	if (shareiofd < 0)
+		err(1, "Could not open '%s'", "/dev/xenshare");
+
+	shareget.share_ref = share_ref;
+	shareget.num_pages = num_pages;
+	ret = ioctl(shareiofd, IOCTL_XENSHARE_GET_SHARE, &shareget);
+	if (ret < 0)
+		err(1, "Getting shared pages gave %i", ret);
+	*peer_id = ret;
+
+	/* Map shared page */
+	sharepage = mmap(NULL, num_pages*getpagesize(), PROT_READ|PROT_WRITE,
+			 MAP_SHARED, shareiofd,
+			 XENSHARE_MAP_SHARE_PAGE * getpagesize());
+	if (sharepage == MAP_FAILED)
+		err(1, "Failed to map shared page");
+
+	return sharepage;
+}
+
+/* Munmap addr, let the xenshare interface clean up evtchns etc */
+static int unmap_pages(void *addr)
+{
+	int err;
+
+	err = munmap((void *)addr, PAGE_SIZE);
+	if (err < 0) {
+		fprintf(stderr, "Failed to munmap() (%i,%i)\n", err, -errno);
+		return -errno;
+	}
+
+	return 0;
+}
+
+/* FIXME: Move to xenctrl library */
+static share_ref_t create_shared_pages(int num_pages, unsigned int *peer_id)
+{
+	share_ref_t share_ref;
+	int err;
+	void *addr;
+
+	dom0_op_t op = { .cmd = DOM0_CREATESHAREDPAGES,
+		 .interface_version = DOM0_INTERFACE_VERSION,
+		 .u.createsharedpages.num = num_pages };
+
+	err = do_dom0_op(xc_fd, &op);
+	if (err < 0)
+		return 0;
+
+	printf("Create page returned 0x%x\n", err);
+
+	/* Save the share_ref */
+	share_ref = err;
+
+	/* Clear the page */
+	addr = map_pages(share_ref, num_pages, peer_id);
+	memset(addr, 0, num_pages * getpagesize());
+	unmap_pages(addr);
+
+	return share_ref;
+}
+
+static uint64_t get_domain_shared_ref(struct xs_handle *h, domid_t domid)
+{
+	unsigned int len;
+	unsigned long long share_ref;
+	char key[512];
+	char *val, *endp;
+
+	sprintf(key, "/local/domain/%i/vdevice-share", domid);
+	val = xs_read(h, 0, key, &len);
+
+	if (val == NULL)
+		return DOMID_FIRST_RESERVED;
+	share_ref = strtoull(val, &endp, 0);
+	if (endp == val || *endp) {
+		errno = EINVAL;
+		free(val);
+		return DOMID_FIRST_RESERVED;
+	}
+	free(val);
+	return share_ref;
+}
+
+/* Get dom0 vdevice_share from /sys/bus/vdevice/share_ref */
+static uint64_t get_dom0_shared_ref(void)
+{
+	FILE *f;
+	unsigned long long share_ref;
+
+	f = fopen("/sys/bus/vdevice/share_ref", "r");
+	if (!f) {
+		return DOMID_FIRST_RESERVED;
+	}
+	if (fscanf(f, "%llx", &share_ref) != 1) {
+		errno = EINVAL;
+		return DOMID_FIRST_RESERVED;
+	}
+	fclose(f);
+	return share_ref;
+}
+
+struct vdevice_type
+{
+	/* Name of this device */
+	const char *name;
+
+	/* Number of pages to create for it. */
+	unsigned int num_pages;
+
+	/* Type number of this device. */
+	uint32_t type;
+
+	/* Features when creating a new one of these */
+	uint32_t features;
+
+	/* --create.  Returns num args consumed. */
+	int (*create)(struct vdevice_type *,
+		      share_ref_t ref, void *map, int argc, char *argv[]);
+
+	/* List info about this vdevice. */
+	void (*list)(struct vdevice_type *, const struct vdevice_desc *vdesc);
+};
+
+/* Volatile is important: someone else changes it. */
+static uint32_t get_status(volatile struct vdevice_desc *vdevice)
+{
+	return vdevice->status;
+}
+
+/* Returns the vdevice reference for this domain. */
+static share_ref_t vdevice_ref_for_domain(domid_t domid)
+{
+	share_ref_t vdevice_ref;
+
+	if (domid == 0)
+		vdevice_ref = get_dom0_shared_ref();
+	else {
+		int saved_errno;
+		struct xs_handle *xsh = xs_daemon_open();
+		if (!xsh) {
+			warn("Could not talk to xenstored");
+			return DOMID_FIRST_RESERVED;
+		}
+		vdevice_ref = get_domain_shared_ref(xsh, domid);
+		saved_errno = errno;
+		xs_daemon_close(xsh);
+		errno = saved_errno;
+	}
+	return vdevice_ref;
+}
+
+static bool add_vdevice_entry(const char *domain,
+			      uint32_t type, uint32_t features,
+			      unsigned int num_pages, share_ref_t share_ref,
+			      uint32_t status_flags)
+{
+	struct vdevice_desc *vdevices;
+	unsigned int i, peer_id;
+	uint32_t status;
+	share_ref_t vdevice_ref;
+	long domid;
+	char *endp;
+
+	domid = strtol(domain, &endp, 0);
+	if (domid >= DOMID_FIRST_RESERVED || endp == domain || *endp != '\0') {
+		warn("Invalid domain id '%s'", domain);
+		return false;
+	}
+
+	vdevice_ref = vdevice_ref_for_domain(domid);
+	if (vdevice_ref == DOMID_FIRST_RESERVED) {
+		warnx("Could not find vdevice page for domain %li", domid);
+		return false;
+	}
+
+	/* There is always excatly 1 page for vdevices */
+	vdevices = map_pages(vdevice_ref, 1, &peer_id);
+	if (!vdevices) {
+		warn("Could not access vdevice page %#llx for domain %li",
+		     (long long)vdevice_ref, domid);
+		return false;
+	}
+
+	for (i = 0; vdevices[i].id.type; i++) {
+		if (i == (PAGE_SIZE / sizeof(struct vdevice_desc)) - 1) {
+			warnx("Vdevice page for domain %li is full", domid);
+			unmap_pages(vdevices);
+			return false;
+		}
+	}
+
+	if (add_grant(share_ref, domid) != 0) {
+		warn("Could not grant domain %li access to device", domid);
+		unmap_pages(vdevices);
+		return false;
+	}
+
+	vdevices[i].id.type = type;
+	vdevices[i].id.features = features;
+	vdevices[i].nr_pages = num_pages;
+	vdevices[i].shared_ref = share_ref;
+	vdevices[i].status = 0;
+
+	/* FIXME: magic "1" */
+	HYPERVISOR_share(XEN_SHARE_trigger, vdevice_ref, 1, 0, 0);
+
+	/* FIXME: Use /dev/xenshare, rather than spinning.  Timeout. */
+	do {
+		status = get_status(&vdevices[i]);
+		sleep(1);
+	} while ((status & (VDEVICE_S_FAILED|status_flags)) == 0);
+
+	if (status & VDEVICE_S_FAILED) {
+		warnx("Adding device %i to domain %li failed: status %#08x",
+		      i, domid, status);
+		/* if add_device filed the shared page is destroyed */
+		vdevices[i].id.type = 0;
+		unmap_pages(vdevices);
+		return false;
+	}
+	unmap_pages(vdevices);
+	return true;
+}
+
+static void remove_vdevice_entry(share_ref_t vdevice_ref,
+				 struct vdevice_type *type,
+				 share_ref_t share_ref)
+{
+	struct vdevice_desc *vdevices;
+	unsigned int i, peer_id;
+
+	vdevices = map_pages(vdevice_ref, 1, &peer_id);
+	if (!vdevices) {
+		warn("Could not access vdevice page");
+		return;
+	}
+
+	for (i = 0; vdevices[i].shared_ref != share_ref; i++) {
+		if (i == (PAGE_SIZE / sizeof(struct vdevice_desc)) - 1) {
+			warnx("Could not find device %s (%li) in vdevice page",
+			      type->name, share_ref);
+			return;
+		}
+	}
+
+	/* FIXME: report the domid we're talking about! */
+	if (vdevices[i].id.type != type->type) {
+		warnx("Vdevice %i using shared ref %li"
+		      " has wrong type: %i",
+		      i, share_ref, vdevices[i].id.type);
+		return;
+	}
+	memset(&vdevices[i], 0, sizeof(vdevices[i]));
+
+	HYPERVISOR_share(XEN_SHARE_trigger, vdevice_ref, 1, 0, 0);
+	/* FIXME: wait for ack! */
+}
+
+/* FIXME: some callers need to recover, not exit if this fails... */
+static share_ref_t domid_arg(const char *arg)
+{
+	unsigned long domain;
+	char *endp;
+	share_ref_t vdevice_ref;
+
+	domain = strtol(arg, &endp, 0);
+	if (strlen(arg) == 0 || *endp != '\0')
+		errx(1, "Invalid domain id '%s'", arg);
+
+	vdevice_ref = vdevice_ref_for_domain(domain);
+	if (vdevice_ref == DOMID_FIRST_RESERVED)
+		err(1, "Cannot find vdevice page for domain '%s'", arg);
+	return vdevice_ref;
+}
+
+static struct vdevice_type types[] = {
+};
+
+#define ARRAY_SIZE(a) (sizeof(a)/sizeof(a[0]))
+static struct vdevice_type *find_type(const char *type)
+{
+	unsigned int i;
+	for (i = 0; i < ARRAY_SIZE(types); i++)
+		if (!strcmp(types[i].name, type))
+			return &types[i];
+	return NULL;
+}
+static struct vdevice_type *find_type_err(const char *type)
+{
+	if (!find_type(type))
+		errx(1, "unknown type '%s'", type);
+	return find_type(type);
+}
+static struct vdevice_type *find_type_number(unsigned int num)
+{
+	unsigned int i;
+	for (i = 0; i < ARRAY_SIZE(types); i++)
+		if (num == types[i].type)
+			return &types[i];
+	return NULL;
+}
+
+static void usage(void)
+{
+	unsigned int i;
+	fprintf(stderr, "Usage:\n"
+		"\t%s --create <type> ...\n"
+		"\t%s --add <type> <share_ref> <domid>\n"
+		"\t%s --remove <type> <share_ref> <domid>\n"
+		"\t%s --delete <type> <share_ref> ...\n"
+		"\t%s --list <domid>\n",
+		PROGRAM_NAME, PROGRAM_NAME, PROGRAM_NAME, PROGRAM_NAME,
+		PROGRAM_NAME);
+	fprintf(stderr, "Available types:");
+	for (i = 0; i < ARRAY_SIZE(types); i++)
+		fprintf(stderr, " %s", types[i].name);
+	fprintf(stderr, "\n");
+	exit(1);
+}
+
+static void list_devices(share_ref_t vdevice_ref)
+{
+	unsigned int i, peer_id;
+	struct vdevice_desc *vdevices;
+
+	vdevices = map_pages(vdevice_ref, 1, &peer_id);
+	if (!vdevices)
+		err(1, "Could not access vdevice page");
+
+	for (i = 0; i < PAGE_SIZE / sizeof(struct vdevice_desc); i++) {
+		struct vdevice_type *type;
+
+		if (!vdevices[i].id.type)
+			continue;
+
+		type = find_type_number(vdevices[i].id.type);
+		printf("Device %i: %s %#x share=%#llx", i,
+		       type ? type->name : "(unknown)",
+		       vdevices[i].status,
+		       (unsigned long long)vdevices[i].shared_ref);
+		if (type)
+			type->list(type, &vdevices[i]);
+		printf("\n");
+	}
+}
+
+static void destroy_share(share_ref_t share_ref)
+{
+	int olderr = errno;
+	dom0_op_t op = { .cmd = DOM0_DESTROYSHAREDPAGES,
+			 .interface_version = DOM0_INTERFACE_VERSION,
+			 .u.destroysharedpages.share_ref = share_ref };
+	if (do_dom0_op(xc_fd, &op) != 0)
+		warn("Failed to destroy share");
+	errno = olderr;
+}
+
+/* Steal the number of pages from the command line if specified,
+ * otherwise use the default from the type defn. */
+static int get_num_pages(int *argc, char **argv, int num_pages)
+{
+	int i, j;
+
+	for(i=0;i<*argc;i++) {
+		if (strcmp(argv[i], "--num_pages") == 0) {
+			if (i == *argc-1)
+				errx(1, "Specified num_pages at end of args");
+
+			num_pages = atoi(argv[i+1]);
+			if (num_pages <= 0)
+				errx(1, "%s is an invalid number of pages",
+					argv[i+1]);
+
+			for(j=0;j+i+2<*argc;j++) {
+				argv[i+j] = argv[i+j+2];
+			}
+			argv[(*argc)-1] = NULL;
+			argv[(*argc)-2] = NULL;
+			*argc -= 2;
+			break;
+		}
+	}
+
+	return num_pages;
+}
+
+static void create_device(struct vdevice_type *type, int argc, char *argv[])
+{
+	unsigned int peer_id;
+	int argoff;
+	share_ref_t share_ref;
+	void *map;
+	int num_pages = get_num_pages(&argc, argv, type->num_pages);
+
+	share_ref = create_shared_pages(num_pages, &peer_id);
+	if (share_ref == 0)
+		err(1, "Failed to create a new shared page!");
+
+	map = xc_map_foreign_range(xc_fd, DOMID_SELF,
+				   PAGE_SIZE * num_pages,
+				   PROT_READ|PROT_WRITE, share_ref);
+	if (!map) {
+		destroy_share(share_ref);
+		err(1, "Failed to map share %li", share_ref);
+	}
+	argoff = type->create(type, share_ref, map, argc, argv);
+	if (argoff < 0) {
+		destroy_share(share_ref);
+		exit(1);
+	}
+	argc -= argoff;
+	argv += argoff;
+
+	while (argv[0]) {
+		add_vdevice_entry(argv[0], type->type, type->features,
+				  num_pages, share_ref, VDEVICE_S_ACKNOWLEDGE);
+		argv++;
+	}
+}
+
+static void add_device(struct vdevice_type *type,
+		       share_ref_t share_ref,
+		       const char *domain)
+{
+	/* FIXME: get nr_pages from vdesc? */
+	if (!add_vdevice_entry(domain, type->type, type->features,
+			       type->num_pages, share_ref,
+			       VDEVICE_S_ACKNOWLEDGE))
+		exit(1);
+}
+
+static void delete_device(struct vdevice_type *type, share_ref_t share_ref,
+			  int argc, char *argv[])
+{
+	/* Remove domains, then destroy share. */
+	while (argv[0]) {
+		remove_vdevice_entry(domid_arg(argv[0]), type, share_ref);
+		argv++;
+	}
+
+	destroy_share(share_ref);
+}
+
+static void remove_device(struct vdevice_type *type,
+			  share_ref_t share_ref,
+			  share_ref_t vdevices_ref)
+{
+	remove_vdevice_entry(vdevices_ref, type, share_ref);
+}
+
+static uint64_t share_ref_arg(const char *arg)
+{
+	char *endp;
+	uint64_t share_ref = strtoull(arg, &endp, 0);
+
+	if (*endp || endp == arg)
+		errx(1, "Invalid shared reference %s", arg);
+	return share_ref;
+}
+
+
+/* FIXME: Locking!  what prevents 2 (or more) userspace apps clobbering each
+ * others memory? */
+int main(int argc, char *argv[])
+{
+	if (argc < 2)
+		usage();
+
+	xc_fd = xc_interface_open();
+	if (xc_fd < 0)
+		err(1, "Failed to open xc interface");
+
+	if (!strcmp(argv[1], "--list")) {
+		if (argc != 3)
+			usage();
+		list_devices(domid_arg(argv[2]));
+	} else if (!strcmp(argv[1], "--create")) {
+		if (argc < 3)
+			usage();
+		create_device(find_type_err(argv[2]), argc-3, argv+3);
+	} else if (!strcmp(argv[1], "--add")) {
+		if (argc != 5)
+			usage();
+		add_device(find_type_err(argv[2]), share_ref_arg(argv[3]),
+			   argv[4]);
+	} else if (!strcmp(argv[1], "--delete")) {
+		if (argc < 4)
+			usage();
+		delete_device(find_type_err(argv[2]), share_ref_arg(argv[3]),
+			      argc-4, argv+4);
+	} else if (!strcmp(argv[1], "--remove")) {
+		if (argc != 5)
+			usage();
+		remove_device(find_type_err(argv[2]), share_ref_arg(argv[3]),
+			      domid_arg(argv[4]));
+	} else
+		usage();
+	return 0;
+}