/usr/src/usr.sbin/vmd/virtio.c

Bug Summary

File:	src/usr.sbin/vmd/virtio.c
Warning:	line 545, column 5 Potential leak of memory pointed to by 'dev'
Annotated Source Code

Press '?' to see keyboard shortcuts
Show analyzer invocation
clang -cc1 -cc1 -triple amd64-unknown-openbsd7.4 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name virtio.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model pic -pic-level 1 -pic-is-pie -mframe-pointer=all -relaxed-aliasing -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/usr.sbin/vmd/obj -resource-dir /usr/local/llvm16/lib/clang/16 -I /usr/src/usr.sbin/vmd -internal-isystem /usr/local/llvm16/lib/clang/16/include -internal-externc-isystem /usr/include -O2 -fdebug-compilation-dir=/usr/src/usr.sbin/vmd/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fcf-protection=branch -fno-jump-tables -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /home/ben/Projects/scan/2024-01-11-140451-98009-1 -x c /usr/src/usr.sbin/vmd/virtio.c
1/*	$OpenBSD: virtio.c,v 1.110 2023/11/03 11:16:43 dv Exp $	*/

3/*
* Copyright (c) 2015 Mike Larkin <mlarkin@openbsd.org>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/

19#include <sys/param.h>	/* PAGE_SIZE */
20#include <sys/socket.h>
21#include <sys/wait.h>

23#include <machine/vmmvar.h>
24#include <dev/pci/pcireg.h>
25#include <dev/pci/pcidevs.h>
26#include <dev/pv/virtioreg.h>
27#include <dev/pci/virtio_pcireg.h>
28#include <dev/pv/vioblkreg.h>
29#include <dev/pv/vioscsireg.h>

31#include <net/if.h>
32#include <netinet/in.h>
33#include <netinet/if_ether.h>
34#include <netinet/ip.h>

36#include <errno(*__errno()).h>
37#include <event.h>
38#include <fcntl.h>
39#include <poll.h>
40#include <stddef.h>
41#include <stdlib.h>
42#include <string.h>
43#include <unistd.h>

45#include "atomicio.h"
46#include "pci.h"
47#include "vioscsi.h"
48#include "virtio.h"
49#include "vmd.h"
50#include "vmm.h"

52extern struct vmd *env;
53extern char *__progname;

55struct viornd_dev viornd;
56struct vioscsi_dev *vioscsi;
57struct vmmci_dev vmmci;

59/* Devices emulated in subprocesses are inserted into this list. */
60SLIST_HEAD(virtio_dev_head, virtio_dev)struct virtio_dev_head { struct virtio_dev *slh_first; } virtio_devs;

62#define MAXPHYS(64 * 1024)	(64 * 1024)	/* max raw I/O transfer size */

64#define VIRTIO_NET_F_MAC(1<<5)	(1<<5)

66#define VMMCI_F_TIMESYNC(1<<0)	(1<<0)
67#define VMMCI_F_ACK(1<<1)		(1<<1)
68#define VMMCI_F_SYNCRTC(1<<2)		(1<<2)

70#define RXQ0	0
71#define TXQ1	1

73static int virtio_dev_launch(struct vmd_vm *, struct virtio_dev *);
74static void virtio_dispatch_dev(int, short, void *);
75static int handle_dev_msg(struct viodev_msg *, struct virtio_dev *);

77const char *
78virtio_reg_name(uint8_t reg)
79{
switch (reg) {
case VIRTIO_CONFIG_DEVICE_FEATURES0: return "device feature";
case VIRTIO_CONFIG_GUEST_FEATURES4: return "guest feature";
case VIRTIO_CONFIG_QUEUE_PFN8: return "queue address";
case VIRTIO_CONFIG_QUEUE_SIZE12: return "queue size";
case VIRTIO_CONFIG_QUEUE_SELECT14: return "queue select";
case VIRTIO_CONFIG_QUEUE_NOTIFY16: return "queue notify";
case VIRTIO_CONFIG_DEVICE_STATUS18: return "device status";
case VIRTIO_CONFIG_ISR_STATUS19: return "isr status";
case VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI20...VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI20 + 3:
return "device config 0";
case VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI20 + 4:
case VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI20 + 5:
return "device config 1";
case VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI20 + 8: return "device config 2";
case VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI20 + 12: return "device config 3";
case VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI20 + 16: return "device config 4";
default: return "unknown";
}
99}

101uint32_t
102vring_size(uint32_t vq_size)
103{
uint32_t allocsize1, allocsize2;

/* allocsize1: descriptor table + avail ring + pad */
allocsize1 = VIRTQUEUE_ALIGN(sizeof(struct vring_desc) * vq_size(((sizeof(struct vring_desc) * vq_size + sizeof(uint16_t) * (
+ vq_size))+((4096)-1))& ~((4096)-1))
   + sizeof(uint16_t) * (2 + vq_size))(((sizeof(struct vring_desc) * vq_size + sizeof(uint16_t) * (
+ vq_size))+((4096)-1))& ~((4096)-1));
/* allocsize2: used ring + pad */
allocsize2 = VIRTQUEUE_ALIGN(sizeof(uint16_t) * 2(((sizeof(uint16_t) * 2 + sizeof(struct vring_used_elem) * vq_size
)+((4096)-1))& ~((4096)-1))
   + sizeof(struct vring_used_elem) * vq_size)(((sizeof(uint16_t) * 2 + sizeof(struct vring_used_elem) * vq_size
)+((4096)-1))& ~((4096)-1));

return allocsize1 + allocsize2;
114}

116/* Update queue select */
117void
118viornd_update_qs(void)
119{
struct virtio_vq_info *vq_info;

/* Invalid queue? */
if (viornd.cfg.queue_select > 0) {
viornd.cfg.queue_size = 0;
return;
}

vq_info = &viornd.vq[viornd.cfg.queue_select];

/* Update queue pfn/size based on queue select */
viornd.cfg.queue_pfn = vq_info->q_gpa >> 12;
viornd.cfg.queue_size = vq_info->qs;
133}

135/* Update queue address */
136void
137viornd_update_qa(void)
138{
struct virtio_vq_info *vq_info;
void *hva = NULL((void *)0);

/* Invalid queue? */
if (viornd.cfg.queue_select > 0)
return;

vq_info = &viornd.vq[viornd.cfg.queue_select];
vq_info->q_gpa = (uint64_t)viornd.cfg.queue_pfn * VIRTIO_PAGE_SIZE(4096);

hva = hvaddr_mem(vq_info->q_gpa, vring_size(VIORND_QUEUE_SIZE64));
if (hva == NULL((void *)0))
fatalx("viornd_update_qa");
vq_info->q_hva = hva;
153}

155int
156viornd_notifyq(void)
157{
size_t sz;
int dxx, ret;
uint16_t aidx, uidx;
char *vr, *rnd_data;
struct vring_desc *desc;
struct vring_avail *avail;
struct vring_used *used;
struct virtio_vq_info *vq_info;

ret = 0;

/* Invalid queue? */
if (viornd.cfg.queue_notify > 0)
return (0);

vq_info = &viornd.vq[viornd.cfg.queue_notify];
vr = vq_info->q_hva;
if (vr == NULL((void *)0))
fatalx("%s: null vring", __func__);

desc = (struct vring_desc *)(vr);
avail = (struct vring_avail *)(vr + vq_info->vq_availoffset);
used = (struct vring_used *)(vr + vq_info->vq_usedoffset);

aidx = avail->idx & VIORND_QUEUE_MASK(64 - 1);
uidx = used->idx & VIORND_QUEUE_MASK(64 - 1);

dxx = avail->ring[aidx] & VIORND_QUEUE_MASK(64 - 1);

sz = desc[dxx].len;
if (sz > MAXPHYS(64 * 1024))
fatalx("viornd descriptor size too large (%zu)", sz);

rnd_data = malloc(sz);

if (rnd_data != NULL((void *)0)) {
arc4random_buf(rnd_data, sz);
if (write_mem(desc[dxx].addr, rnd_data, sz)) {
	log_warnx("viornd: can't write random data @ "
	    "0x%llx",
	    desc[dxx].addr);
} else {
	/* ret == 1 -> interrupt needed */
	/* XXX check VIRTIO_F_NO_INTR */
	ret = 1;
	viornd.cfg.isr_status = 1;
	used->ring[uidx].id = dxx;
	used->ring[uidx].len = sz;
	__sync_synchronize();
	used->idx++;
}
free(rnd_data);
} else
fatal("memory allocation error for viornd data");

return (ret);
214}

216int
217virtio_rnd_io(int dir, uint16_t reg, uint32_t *data, uint8_t *intr,
  void *unused, uint8_t sz)
219{
*intr = 0xFF;

if (dir == 0) {
switch (reg) {
case VIRTIO_CONFIG_DEVICE_FEATURES0:
case VIRTIO_CONFIG_QUEUE_SIZE12:
case VIRTIO_CONFIG_ISR_STATUS19:
	log_warnx("%s: illegal write %x to %s",
	    __progname, *data, virtio_reg_name(reg));
	break;
case VIRTIO_CONFIG_GUEST_FEATURES4:
	viornd.cfg.guest_feature = *data;
	break;
case VIRTIO_CONFIG_QUEUE_PFN8:
	viornd.cfg.queue_pfn = *data;
	viornd_update_qa();
	break;
case VIRTIO_CONFIG_QUEUE_SELECT14:
	viornd.cfg.queue_select = *data;
	viornd_update_qs();
	break;
case VIRTIO_CONFIG_QUEUE_NOTIFY16:
	viornd.cfg.queue_notify = *data;
	if (viornd_notifyq())
		*intr = 1;
	break;
case VIRTIO_CONFIG_DEVICE_STATUS18:
	viornd.cfg.device_status = *data;
	break;
}
} else {
switch (reg) {
case VIRTIO_CONFIG_DEVICE_FEATURES0:
	*data = viornd.cfg.device_feature;
	break;
case VIRTIO_CONFIG_GUEST_FEATURES4:
	*data = viornd.cfg.guest_feature;
	break;
case VIRTIO_CONFIG_QUEUE_PFN8:
	*data = viornd.cfg.queue_pfn;
	break;
case VIRTIO_CONFIG_QUEUE_SIZE12:
	*data = viornd.cfg.queue_size;
	break;
case VIRTIO_CONFIG_QUEUE_SELECT14:
	*data = viornd.cfg.queue_select;
	break;
case VIRTIO_CONFIG_QUEUE_NOTIFY16:
	*data = viornd.cfg.queue_notify;
	break;
case VIRTIO_CONFIG_DEVICE_STATUS18:
	*data = viornd.cfg.device_status;
	break;
case VIRTIO_CONFIG_ISR_STATUS19:
	*data = viornd.cfg.isr_status;
	viornd.cfg.isr_status = 0;
	vcpu_deassert_pic_irq(viornd.vm_id, 0, viornd.irq);
	break;
}
}
return (0);
281}

283int
284vmmci_ctl(unsigned int cmd)
285{
struct timeval tv = { 0, 0 };

if ((vmmci.cfg.device_status &
   VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK4) == 0)
return (-1);

if (cmd == vmmci.cmd)
return (0);

switch (cmd) {
case VMMCI_NONE:
break;
case VMMCI_SHUTDOWN:
case VMMCI_REBOOT:
/* Update command */
vmmci.cmd = cmd;

/*
 * vmm VMs do not support powerdown, send a reboot request
 * instead and turn it off after the triple fault.
 */
if (cmd == VMMCI_SHUTDOWN)
	cmd = VMMCI_REBOOT;

/* Trigger interrupt */
vmmci.cfg.isr_status = VIRTIO_CONFIG_ISR_CONFIG_CHANGE2;
vcpu_assert_pic_irq(vmmci.vm_id, 0, vmmci.irq);

/* Add ACK timeout */
tv.tv_sec = VMMCI_TIMEOUT3;
evtimer_add(&vmmci.timeout, &tv)event_add(&vmmci.timeout, &tv);
break;
case VMMCI_SYNCRTC:
if (vmmci.cfg.guest_feature & VMMCI_F_SYNCRTC(1<<2)) {
	/* RTC updated, request guest VM resync of its RTC */
	vmmci.cmd = cmd;

	vmmci.cfg.isr_status = VIRTIO_CONFIG_ISR_CONFIG_CHANGE2;
	vcpu_assert_pic_irq(vmmci.vm_id, 0, vmmci.irq);
} else {
	log_debug("%s: RTC sync skipped (guest does not "
	    "support RTC sync)\n", __func__);
}
break;
default:
fatalx("invalid vmmci command: %d", cmd);
}

return (0);
335}

337void
338vmmci_ack(unsigned int cmd)
339{
struct timeval	 tv = { 0, 0 };

switch (cmd) {
case VMMCI_NONE:
break;
case VMMCI_SHUTDOWN:
/*
 * The shutdown was requested by the VM if we don't have
 * a pending shutdown request.  In this case add a short
 * timeout to give the VM a chance to reboot before the
 * timer is expired.
 */
if (vmmci.cmd == 0) {
	log_debug("%s: vm %u requested shutdown", __func__,
	    vmmci.vm_id);
	tv.tv_sec = VMMCI_TIMEOUT3;
	evtimer_add(&vmmci.timeout, &tv)event_add(&vmmci.timeout, &tv);
	return;
}
/* FALLTHROUGH */
case VMMCI_REBOOT:
/*
 * If the VM acknowledged our shutdown request, give it
 * enough time to shutdown or reboot gracefully.  This
 * might take a considerable amount of time (running
 * rc.shutdown on the VM), so increase the timeout before
 * killing it forcefully.
 */
if (cmd == vmmci.cmd &&
    evtimer_pending(&vmmci.timeout, NULL)event_pending(&vmmci.timeout, 0x01, ((void *)0))) {
	log_debug("%s: vm %u acknowledged shutdown request",
	    __func__, vmmci.vm_id);
	tv.tv_sec = VMMCI_SHUTDOWN_TIMEOUT120;
	evtimer_add(&vmmci.timeout, &tv)event_add(&vmmci.timeout, &tv);
}
break;
case VMMCI_SYNCRTC:
log_debug("%s: vm %u acknowledged RTC sync request",
    __func__, vmmci.vm_id);
vmmci.cmd = VMMCI_NONE;
break;
default:
log_warnx("%s: illegal request %u", __func__, cmd);
break;
}
385}

387void
388vmmci_timeout(int fd, short type, void *arg)
389{
log_debug("%s: vm %u shutdown", __progname, vmmci.vm_id);
vm_shutdown(vmmci.cmd == VMMCI_REBOOT ? VMMCI_REBOOT : VMMCI_SHUTDOWN);
392}

394int
395vmmci_io(int dir, uint16_t reg, uint32_t *data, uint8_t *intr,
  void *unused, uint8_t sz)
397{
*intr = 0xFF;

if (dir == 0) {
switch (reg) {
case VIRTIO_CONFIG_DEVICE_FEATURES0:
case VIRTIO_CONFIG_QUEUE_SIZE12:
case VIRTIO_CONFIG_ISR_STATUS19:
	log_warnx("%s: illegal write %x to %s",
	    __progname, *data, virtio_reg_name(reg));
	break;
case VIRTIO_CONFIG_GUEST_FEATURES4:
	vmmci.cfg.guest_feature = *data;
	break;
case VIRTIO_CONFIG_QUEUE_PFN8:
	vmmci.cfg.queue_pfn = *data;
	break;
case VIRTIO_CONFIG_QUEUE_SELECT14:
	vmmci.cfg.queue_select = *data;
	break;
case VIRTIO_CONFIG_QUEUE_NOTIFY16:
	vmmci.cfg.queue_notify = *data;
	break;
case VIRTIO_CONFIG_DEVICE_STATUS18:
	vmmci.cfg.device_status = *data;
	break;
case VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI20:
	vmmci_ack(*data);
	break;
}
} else {
switch (reg) {
case VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI20:
	*data = vmmci.cmd;
	break;
case VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI20 + 4:
	/* Update time once when reading the first register */
	gettimeofday(&vmmci.time, NULL((void *)0));
	*data = (uint64_t)vmmci.time.tv_sec;
	break;
case VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI20 + 8:
	*data = (uint64_t)vmmci.time.tv_sec << 32;
	break;
case VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI20 + 12:
	*data = (uint64_t)vmmci.time.tv_usec;
	break;
case VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI20 + 16:
	*data = (uint64_t)vmmci.time.tv_usec << 32;
	break;
case VIRTIO_CONFIG_DEVICE_FEATURES0:
	*data = vmmci.cfg.device_feature;
	break;
case VIRTIO_CONFIG_GUEST_FEATURES4:
	*data = vmmci.cfg.guest_feature;
	break;
case VIRTIO_CONFIG_QUEUE_PFN8:
	*data = vmmci.cfg.queue_pfn;
	break;
case VIRTIO_CONFIG_QUEUE_SIZE12:
	*data = vmmci.cfg.queue_size;
	break;
case VIRTIO_CONFIG_QUEUE_SELECT14:
	*data = vmmci.cfg.queue_select;
	break;
case VIRTIO_CONFIG_QUEUE_NOTIFY16:
	*data = vmmci.cfg.queue_notify;
	break;
case VIRTIO_CONFIG_DEVICE_STATUS18:
	*data = vmmci.cfg.device_status;
	break;
case VIRTIO_CONFIG_ISR_STATUS19:
	*data = vmmci.cfg.isr_status;
	vmmci.cfg.isr_status = 0;
	vcpu_deassert_pic_irq(vmmci.vm_id, 0, vmmci.irq);
	break;
}
}
return (0);
475}

477int
478virtio_get_base(int fd, char *path, size_t npath, int type, const char *dpath)
479{
switch (type) {
case VMDF_RAW0x01:
return 0;
case VMDF_QCOW20x02:
return virtio_qcow2_get_base(fd, path, npath, dpath);
}
log_warnx("%s: invalid disk format", __func__);
return -1;
488}

490void
491virtio_init(struct vmd_vm *vm, int child_cdrom,
  int child_disks[][VM_MAX_BASE_PER_DISK4], int *child_taps)
493{
struct vmop_create_params *vmc = &vm->vm_params;
struct vm_create_params *vcp = &vmc->vmc_params;
struct virtio_dev *dev;
uint8_t id;
uint8_t i, j;

/* Virtio entropy device */
if (pci_add_device(&id, PCI_VENDOR_QUMRANET0x1af4,
1
Assuming the condition is false→
2
←
Taking false branch→
   PCI_PRODUCT_QUMRANET_VIO_RNG0x1005, PCI_CLASS_SYSTEM0x08,
   PCI_SUBCLASS_SYSTEM_MISC0x80,
   PCI_VENDOR_OPENBSD0x0b5d,
   PCI_PRODUCT_VIRTIO_ENTROPY4, 1, NULL((void *)0))) {
log_warnx("%s: can't add PCI virtio rng device",
    __progname);
return;
}

if (pci_add_bar(id, PCI_MAPREG_TYPE_IO0x00000001, virtio_rnd_io, NULL((void *)0))) {
3
←
Assuming the condition is false→
4
←
Taking false branch→
log_warnx("%s: can't add bar for virtio rng device",
    __progname);
return;
}

memset(&viornd, 0, sizeof(viornd));
viornd.vq[0].qs = VIORND_QUEUE_SIZE64;
viornd.vq[0].vq_availoffset = sizeof(struct vring_desc) *
   VIORND_QUEUE_SIZE64;
viornd.vq[0].vq_usedoffset = VIRTQUEUE_ALIGN((((sizeof(struct vring_desc) * 64 + sizeof(uint16_t) * (2 + 64
))+((4096)-1))& ~((4096)-1))
   sizeof(struct vring_desc) * VIORND_QUEUE_SIZE(((sizeof(struct vring_desc) * 64 + sizeof(uint16_t) * (2 + 64
))+((4096)-1))& ~((4096)-1))
   + sizeof(uint16_t) * (2 + VIORND_QUEUE_SIZE))(((sizeof(struct vring_desc) * 64 + sizeof(uint16_t) * (2 + 64
))+((4096)-1))& ~((4096)-1));
viornd.pci_id = id;
viornd.irq = pci_get_dev_irq(id);
viornd.vm_id = vcp->vcp_id;

SLIST_INIT(&virtio_devs){ ((&virtio_devs)->slh_first) = ((void *)0); };

if (vmc->vmc_nnics > 0) {
5
←
Assuming field 'vmc_nnics' is > 0→
6
←
Taking true branch→
for (i = 0; i6.1
'i' is < field 'vmc_nnics'
 < vmc->vmc_nnics; i++) {
7
←
Loop condition is true.  Entering loop body→
	dev = calloc(1, sizeof(struct virtio_dev));
8
←
Memory is allocated→
	if (dev == NULL((void *)0)) {
9
←
Assuming 'dev' is not equal to NULL→
10
←
Taking false branch→
		log_warn("%s: calloc failure allocating vionet",
		    __progname);
		return;
	}
	/* Virtio network */
	dev->dev_type = VMD_DEVTYPE_NET'n';

	if (pci_add_device(&id, PCI_VENDOR_QUMRANET0x1af4,
11
←
Assuming the condition is true→
12
←
Taking true branch→
		PCI_PRODUCT_QUMRANET_VIO_NET0x1000, PCI_CLASS_SYSTEM0x08,
		PCI_SUBCLASS_SYSTEM_MISC0x80, PCI_VENDOR_OPENBSD0x0b5d,
		PCI_PRODUCT_VIRTIO_NETWORK1, 1, NULL((void *)0))) {
		log_warnx("%s: can't add PCI virtio net device",
13
←
Potential leak of memory pointed to by 'dev'
		    __progname);
		return;
	}
	dev->pci_id = id;
	dev->sync_fd = -1;
	dev->async_fd = -1;
	dev->vm_id = vcp->vcp_id;
	dev->vm_vmid = vm->vm_vmid;
	dev->irq = pci_get_dev_irq(id);

	/* The vionet pci bar function is called by the vcpu. */
	if (pci_add_bar(id, PCI_MAPREG_TYPE_IO0x00000001, virtio_pci_io,
	    dev)) {
		log_warnx("%s: can't add bar for virtio net "
		    "device", __progname);
		return;
	}

	dev->vionet.vq[RXQ0].qs = VIONET_QUEUE_SIZE256;
	dev->vionet.vq[RXQ0].vq_availoffset =
	    sizeof(struct vring_desc) * VIONET_QUEUE_SIZE256;
	dev->vionet.vq[RXQ0].vq_usedoffset = VIRTQUEUE_ALIGN((((sizeof(struct vring_desc) * 256 + sizeof(uint16_t) * (2 + 256
))+((4096)-1))& ~((4096)-1))
		sizeof(struct vring_desc) * VIONET_QUEUE_SIZE(((sizeof(struct vring_desc) * 256 + sizeof(uint16_t) * (2 + 256
))+((4096)-1))& ~((4096)-1))
		+ sizeof(uint16_t) * (2 + VIONET_QUEUE_SIZE))(((sizeof(struct vring_desc) * 256 + sizeof(uint16_t) * (2 + 256
))+((4096)-1))& ~((4096)-1));
	dev->vionet.vq[RXQ0].last_avail = 0;
	dev->vionet.vq[RXQ0].notified_avail = 0;

	dev->vionet.vq[TXQ1].qs = VIONET_QUEUE_SIZE256;
	dev->vionet.vq[TXQ1].vq_availoffset =
	    sizeof(struct vring_desc) * VIONET_QUEUE_SIZE256;
	dev->vionet.vq[TXQ1].vq_usedoffset = VIRTQUEUE_ALIGN((((sizeof(struct vring_desc) * 256 + sizeof(uint16_t) * (2 + 256
))+((4096)-1))& ~((4096)-1))
		sizeof(struct vring_desc) * VIONET_QUEUE_SIZE(((sizeof(struct vring_desc) * 256 + sizeof(uint16_t) * (2 + 256
))+((4096)-1))& ~((4096)-1))
		+ sizeof(uint16_t) * (2 + VIONET_QUEUE_SIZE))(((sizeof(struct vring_desc) * 256 + sizeof(uint16_t) * (2 + 256
))+((4096)-1))& ~((4096)-1));
	dev->vionet.vq[TXQ1].last_avail = 0;
	dev->vionet.vq[TXQ1].notified_avail = 0;

	dev->vionet.data_fd = child_taps[i];

	/* MAC address has been assigned by the parent */
	memcpy(&dev->vionet.mac, &vmc->vmc_macs[i], 6);
	dev->vionet.cfg.device_feature = VIRTIO_NET_F_MAC(1<<5);

	dev->vionet.lockedmac =
	    vmc->vmc_ifflags[i] & VMIFF_LOCKED0x02 ? 1 : 0;
	dev->vionet.local =
	    vmc->vmc_ifflags[i] & VMIFF_LOCAL0x04 ? 1 : 0;
	if (i == 0 && vmc->vmc_bootdevice & VMBOOTDEV_NET3)
		dev->vionet.pxeboot = 1;
	memcpy(&dev->vionet.local_prefix,
	    &env->vmd_cfg.cfg_localprefix,
	    sizeof(dev->vionet.local_prefix));
	log_debug("%s: vm \"%s\" vio%u lladdr %s%s%s%s",
	    __func__, vcp->vcp_name, i,
	    ether_ntoa((void *)dev->vionet.mac),
	    dev->vionet.lockedmac ? ", locked" : "",
	    dev->vionet.local ? ", local" : "",
	    dev->vionet.pxeboot ? ", pxeboot" : "");

	/* Add the vionet to our device list. */
	dev->vionet.idx = i;
	SLIST_INSERT_HEAD(&virtio_devs, dev, dev_next)do { (dev)->dev_next.sle_next = (&virtio_devs)->slh_first
; (&virtio_devs)->slh_first = (dev); } while (0);
}
}

if (vmc->vmc_ndisks > 0) {
for (i = 0; i < vmc->vmc_ndisks; i++) {
	dev = calloc(1, sizeof(struct virtio_dev));
	if (dev == NULL((void *)0)) {
		log_warn("%s: calloc failure allocating vioblk",
		    __progname);
		return;
	}

	/* One vioblk device for each disk defined in vcp */
	dev->dev_type = VMD_DEVTYPE_DISK'd';

	if (pci_add_device(&id, PCI_VENDOR_QUMRANET0x1af4,
	    PCI_PRODUCT_QUMRANET_VIO_BLOCK0x1001,
	    PCI_CLASS_MASS_STORAGE0x01,
	    PCI_SUBCLASS_MASS_STORAGE_SCSI0x00,
	    PCI_VENDOR_OPENBSD0x0b5d,
	    PCI_PRODUCT_VIRTIO_BLOCK2, 1, NULL((void *)0))) {
		log_warnx("%s: can't add PCI virtio block "
		    "device", __progname);
		return;
	}
	dev->pci_id = id;
	dev->sync_fd = -1;
	dev->async_fd = -1;
	dev->vm_id = vcp->vcp_id;
	dev->vm_vmid = vm->vm_vmid;
	dev->irq = pci_get_dev_irq(id);

	if (pci_add_bar(id, PCI_MAPREG_TYPE_IO0x00000001, virtio_pci_io,
	    &dev->vioblk)) {
		log_warnx("%s: can't add bar for virtio block "
		    "device", __progname);
		return;
	}
	dev->vioblk.vq[0].qs = VIOBLK_QUEUE_SIZE128;
	dev->vioblk.vq[0].vq_availoffset =
	    sizeof(struct vring_desc) * VIOBLK_QUEUE_SIZE128;
	dev->vioblk.vq[0].vq_usedoffset = VIRTQUEUE_ALIGN((((sizeof(struct vring_desc) * 128 + sizeof(uint16_t) * (2 + 128
))+((4096)-1))& ~((4096)-1))
	    sizeof(struct vring_desc) * VIOBLK_QUEUE_SIZE(((sizeof(struct vring_desc) * 128 + sizeof(uint16_t) * (2 + 128
))+((4096)-1))& ~((4096)-1))
	    + sizeof(uint16_t) * (2 + VIOBLK_QUEUE_SIZE))(((sizeof(struct vring_desc) * 128 + sizeof(uint16_t) * (2 + 128
))+((4096)-1))& ~((4096)-1));
	dev->vioblk.vq[0].last_avail = 0;
	dev->vioblk.cfg.device_feature =
	    VIRTIO_BLK_F_SEG_MAX(1ULL<<2);
	dev->vioblk.seg_max = VIOBLK_SEG_MAX(128 - 2);

	/*
	 * Initialize disk fds to an invalid fd (-1), then
	 * set any child disk fds.
	 */
	memset(&dev->vioblk.disk_fd, -1,
	    sizeof(dev->vioblk.disk_fd));
	dev->vioblk.ndisk_fd = vmc->vmc_diskbases[i];
	for (j = 0; j < dev->vioblk.ndisk_fd; j++)
		dev->vioblk.disk_fd[j] = child_disks[i][j];

	dev->vioblk.idx = i;
	SLIST_INSERT_HEAD(&virtio_devs, dev, dev_next)do { (dev)->dev_next.sle_next = (&virtio_devs)->slh_first
; (&virtio_devs)->slh_first = (dev); } while (0);
}
}

/*
* Launch virtio devices that support subprocess execution.
*/
SLIST_FOREACH(dev, &virtio_devs, dev_next)for((dev) = ((&virtio_devs)->slh_first); (dev) != ((void
 *)0); (dev) = ((dev)->dev_next.sle_next)) {
if (virtio_dev_launch(vm, dev) != 0)
	fatalx("failed to launch virtio device");
}

/* vioscsi cdrom */
if (strlen(vmc->vmc_cdrom)) {
vioscsi = calloc(1, sizeof(struct vioscsi_dev));
if (vioscsi == NULL((void *)0)) {
	log_warn("%s: calloc failure allocating vioscsi",
	    __progname);
	return;
}

if (pci_add_device(&id, PCI_VENDOR_QUMRANET0x1af4,
    PCI_PRODUCT_QUMRANET_VIO_SCSI0x1004,
    PCI_CLASS_MASS_STORAGE0x01,
    PCI_SUBCLASS_MASS_STORAGE_SCSI0x00,
    PCI_VENDOR_OPENBSD0x0b5d,
    PCI_PRODUCT_VIRTIO_SCSI8, 1, NULL((void *)0))) {
	log_warnx("%s: can't add PCI vioscsi device",
	    __progname);
	return;
}

if (pci_add_bar(id, PCI_MAPREG_TYPE_IO0x00000001, vioscsi_io, vioscsi)) {
	log_warnx("%s: can't add bar for vioscsi device",
	    __progname);
	return;
}

for (i = 0; i < VIRTIO_MAX_QUEUES3; i++) {
	vioscsi->vq[i].qs = VIOSCSI_QUEUE_SIZE128;
	vioscsi->vq[i].vq_availoffset =
	    sizeof(struct vring_desc) * VIOSCSI_QUEUE_SIZE128;
	vioscsi->vq[i].vq_usedoffset = VIRTQUEUE_ALIGN((((sizeof(struct vring_desc) * 128 + sizeof(uint16_t) * (2 + 128
))+((4096)-1))& ~((4096)-1))
	    sizeof(struct vring_desc) * VIOSCSI_QUEUE_SIZE(((sizeof(struct vring_desc) * 128 + sizeof(uint16_t) * (2 + 128
))+((4096)-1))& ~((4096)-1))
	    + sizeof(uint16_t) * (2 + VIOSCSI_QUEUE_SIZE))(((sizeof(struct vring_desc) * 128 + sizeof(uint16_t) * (2 + 128
))+((4096)-1))& ~((4096)-1));
	vioscsi->vq[i].last_avail = 0;
}
if (virtio_raw_init(&vioscsi->file, &vioscsi->sz, &child_cdrom,
    1) == -1) {
	log_warnx("%s: unable to determine iso format",
	    __func__);
	return;
}
vioscsi->locked = 0;
vioscsi->lba = 0;
vioscsi->n_blocks = vioscsi->sz / VIOSCSI_BLOCK_SIZE_CDROM2048;
vioscsi->max_xfer = VIOSCSI_BLOCK_SIZE_CDROM2048;
vioscsi->pci_id = id;
vioscsi->vm_id = vcp->vcp_id;
vioscsi->irq = pci_get_dev_irq(id);
}

/* virtio control device */
if (pci_add_device(&id, PCI_VENDOR_OPENBSD0x0b5d,
   PCI_PRODUCT_OPENBSD_CONTROL0x0777,
   PCI_CLASS_COMMUNICATIONS0x07,
   PCI_SUBCLASS_COMMUNICATIONS_MISC0x80,
   PCI_VENDOR_OPENBSD0x0b5d,
   PCI_PRODUCT_VIRTIO_VMMCI65535, 1, NULL((void *)0))) {
log_warnx("%s: can't add PCI vmm control device",
    __progname);
return;
}

if (pci_add_bar(id, PCI_MAPREG_TYPE_IO0x00000001, vmmci_io, NULL((void *)0))) {
log_warnx("%s: can't add bar for vmm control device",
    __progname);
return;
}

memset(&vmmci, 0, sizeof(vmmci));
vmmci.cfg.device_feature = VMMCI_F_TIMESYNC(1<<0) | VMMCI_F_ACK(1<<1) |
   VMMCI_F_SYNCRTC(1<<2);
vmmci.vm_id = vcp->vcp_id;
vmmci.irq = pci_get_dev_irq(id);
vmmci.pci_id = id;

evtimer_set(&vmmci.timeout, vmmci_timeout, NULL)event_set(&vmmci.timeout, -1, 0, vmmci_timeout, ((void *)
0));
755}

757/*
* vionet_set_hostmac
*
* Sets the hardware address for the host-side tap(4) on a vionet_dev.
*
* This should only be called from the event-loop thread
*
* vm: pointer to the current vmd_vm instance
* idx: index into the array of vionet_dev's for the target vionet_dev
* addr: ethernet address to set
*/
768void
769vionet_set_hostmac(struct vmd_vm *vm, unsigned int idx, uint8_t *addr)
770{
struct vmop_create_params	*vmc = &vm->vm_params;
struct virtio_dev		*dev;
struct vionet_dev		*vionet = NULL((void *)0);
int ret;

if (idx > vmc->vmc_nnics)
fatalx("%s: invalid vionet index: %u", __func__, idx);

SLIST_FOREACH(dev, &virtio_devs, dev_next)for((dev) = ((&virtio_devs)->slh_first); (dev) != ((void
 *)0); (dev) = ((dev)->dev_next.sle_next)) {
if (dev->dev_type == VMD_DEVTYPE_NET'n'
    && dev->vionet.idx == idx) {
	vionet = &dev->vionet;
	break;
}
}
if (vionet == NULL((void *)0))
fatalx("%s: dev == NULL, idx = %u", __func__, idx);

/* Set the local vm process copy. */
memcpy(vionet->hostmac, addr, sizeof(vionet->hostmac));

/* Send the information to the device process. */
ret = imsg_compose_event(&dev->async_iev, IMSG_DEVOP_HOSTMAC, 0, 0, -1,
   vionet->hostmac, sizeof(vionet->hostmac));
if (ret == -1) {
log_warnx("%s: failed to queue hostmac to vionet dev %u",
    __func__, idx);
return;
}
800}

802void
803virtio_shutdown(struct vmd_vm *vm)
804{
int ret, status;
pid_t pid = 0;
struct virtio_dev *dev, *tmp;
struct viodev_msg msg;
struct imsgbuf *ibuf;

/* Ensure that our disks are synced. */
if (vioscsi != NULL((void *)0))
vioscsi->file.close(vioscsi->file.p, 0);

/*
* Broadcast shutdown to child devices. We need to do this
* synchronously as we have already stopped the async event thread.
*/
SLIST_FOREACH(dev, &virtio_devs, dev_next)for((dev) = ((&virtio_devs)->slh_first); (dev) != ((void
 *)0); (dev) = ((dev)->dev_next.sle_next)) {
memset(&msg, 0, sizeof(msg));
msg.type = VIODEV_MSG_SHUTDOWN7;
ibuf = &dev->sync_iev.ibuf;
ret = imsg_compose(ibuf, VIODEV_MSG_SHUTDOWN7, 0, 0, -1,
    &msg, sizeof(msg));
if (ret == -1)
	fatalx("%s: failed to send shutdown to device",
	    __func__);
if (imsg_flush(ibuf) == -1)
	fatalx("%s: imsg_flush", __func__);
}

/*
* Wait for all children to shutdown using a simple approach of
* iterating over known child devices and waiting for them to die.
*/
SLIST_FOREACH_SAFE(dev, &virtio_devs, dev_next, tmp)for ((dev) = ((&virtio_devs)->slh_first); (dev) &&
 ((tmp) = ((dev)->dev_next.sle_next), 1); (dev) = (tmp)) {
log_debug("%s: waiting on device pid %d", __func__,
    dev->dev_pid);
do {
	pid = waitpid(dev->dev_pid, &status, WNOHANG0x01);
} while (pid == 0 || (pid == -1 && errno(*__errno()) == EINTR4));
if (pid == dev->dev_pid)
	log_debug("%s: device for pid %d is stopped",
	    __func__, pid);
else
	log_warnx("%s: unexpected pid %d", __func__, pid);
free(dev);
}
849}

851int
852vmmci_restore(int fd, uint32_t vm_id)
853{
log_debug("%s: receiving vmmci", __func__);
if (atomicio(read, fd, &vmmci, sizeof(vmmci)) != sizeof(vmmci)) {
log_warnx("%s: error reading vmmci from fd", __func__);
return (-1);
}

if (pci_set_bar_fn(vmmci.pci_id, 0, vmmci_io, NULL((void *)0))) {
log_warnx("%s: can't set bar fn for vmm control device",
    __progname);
return (-1);
}
vmmci.vm_id = vm_id;
vmmci.irq = pci_get_dev_irq(vmmci.pci_id);
memset(&vmmci.timeout, 0, sizeof(struct event));
evtimer_set(&vmmci.timeout, vmmci_timeout, NULL)event_set(&vmmci.timeout, -1, 0, vmmci_timeout, ((void *)
0));
return (0);
870}

872int
873viornd_restore(int fd, struct vmd_vm *vm)
874{
void *hva = NULL((void *)0);

log_debug("%s: receiving viornd", __func__);
if (atomicio(read, fd, &viornd, sizeof(viornd)) != sizeof(viornd)) {
log_warnx("%s: error reading viornd from fd", __func__);
return (-1);
}
if (pci_set_bar_fn(viornd.pci_id, 0, virtio_rnd_io, NULL((void *)0))) {
log_warnx("%s: can't set bar fn for virtio rng device",
    __progname);
return (-1);
}
viornd.vm_id = vm->vm_params.vmc_params.vcp_id;
viornd.irq = pci_get_dev_irq(viornd.pci_id);

hva = hvaddr_mem(viornd.vq[0].q_gpa, vring_size(VIORND_QUEUE_SIZE64));
if (hva == NULL((void *)0))
fatal("failed to restore viornd virtqueue");
viornd.vq[0].q_hva = hva;

return (0);
896}

898int
899vionet_restore(int fd, struct vmd_vm *vm, int *child_taps)
900{
struct vmop_create_params *vmc = &vm->vm_params;
struct vm_create_params *vcp = &vmc->vmc_params;
struct virtio_dev *dev;
uint8_t i;

if (vmc->vmc_nnics == 0)
return (0);

for (i = 0; i < vmc->vmc_nnics; i++) {
dev = calloc(1, sizeof(struct virtio_dev));
if (dev == NULL((void *)0)) {
	log_warn("%s: calloc failure allocating vionet",
	    __progname);
	return (-1);
}

log_debug("%s: receiving virtio network device", __func__);
if (atomicio(read, fd, dev, sizeof(struct virtio_dev))
    != sizeof(struct virtio_dev)) {
	log_warnx("%s: error reading vionet from fd",
	    __func__);
	return (-1);
}

/* Virtio network */
if (dev->dev_type != VMD_DEVTYPE_NET'n') {
	log_warnx("%s: invalid device type", __func__);
	return (-1);
}

dev->sync_fd = -1;
dev->async_fd = -1;
dev->vm_id = vcp->vcp_id;
dev->vm_vmid = vm->vm_vmid;
dev->irq = pci_get_dev_irq(dev->pci_id);

if (pci_set_bar_fn(dev->pci_id, 0, virtio_pci_io, dev)) {
	log_warnx("%s: can't set bar fn for virtio net "
	    "device", __progname);
	return (-1);
}

dev->vionet.data_fd = child_taps[i];
dev->vionet.idx = i;

SLIST_INSERT_HEAD(&virtio_devs, dev, dev_next)do { (dev)->dev_next.sle_next = (&virtio_devs)->slh_first
; (&virtio_devs)->slh_first = (dev); } while (0);
}

return (0);
950}

952int
953vioblk_restore(int fd, struct vmd_vm *vm,
  int child_disks[][VM_MAX_BASE_PER_DISK4])
955{
struct vmop_create_params *vmc = &vm->vm_params;
struct virtio_dev *dev;
uint8_t i, j;

if (vmc->vmc_ndisks == 0)
return (0);

for (i = 0; i < vmc->vmc_ndisks; i++) {
dev = calloc(1, sizeof(struct virtio_dev));
if (dev == NULL((void *)0)) {
	log_warn("%s: calloc failure allocating vioblks",
	    __progname);
	return (-1);
}

log_debug("%s: receiving vioblk", __func__);
if (atomicio(read, fd, dev, sizeof(struct virtio_dev))
    != sizeof(struct virtio_dev)) {
	log_warnx("%s: error reading vioblk from fd", __func__);
	return (-1);
}
if (dev->dev_type != VMD_DEVTYPE_DISK'd') {
	log_warnx("%s: invalid device type", __func__);
	return (-1);
}

dev->sync_fd = -1;
dev->async_fd = -1;

if (pci_set_bar_fn(dev->pci_id, 0, virtio_pci_io, dev)) {
	log_warnx("%s: can't set bar fn for virtio block "
	    "device", __progname);
	return (-1);
}
dev->vm_id = vmc->vmc_params.vcp_id;
dev->irq = pci_get_dev_irq(dev->pci_id);

memset(&dev->vioblk.disk_fd, -1, sizeof(dev->vioblk.disk_fd));
dev->vioblk.ndisk_fd = vmc->vmc_diskbases[i];
for (j = 0; j < dev->vioblk.ndisk_fd; j++)
	dev->vioblk.disk_fd[j] = child_disks[i][j];

dev->vioblk.idx = i;
SLIST_INSERT_HEAD(&virtio_devs, dev, dev_next)do { (dev)->dev_next.sle_next = (&virtio_devs)->slh_first
; (&virtio_devs)->slh_first = (dev); } while (0);
}
return (0);
1002}

1004int
1005vioscsi_restore(int fd, struct vmd_vm *vm, int child_cdrom)
1006{
void *hva = NULL((void *)0);
unsigned int i;

if (!strlen(vm->vm_params.vmc_cdrom))
return (0);

vioscsi = calloc(1, sizeof(struct vioscsi_dev));
if (vioscsi == NULL((void *)0)) {
log_warn("%s: calloc failure allocating vioscsi", __progname);
return (-1);
}

log_debug("%s: receiving vioscsi", __func__);

if (atomicio(read, fd, vioscsi, sizeof(struct vioscsi_dev)) !=
   sizeof(struct vioscsi_dev)) {
log_warnx("%s: error reading vioscsi from fd", __func__);
return (-1);
}

if (pci_set_bar_fn(vioscsi->pci_id, 0, vioscsi_io, vioscsi)) {
log_warnx("%s: can't set bar fn for vmm control device",
    __progname);
return (-1);
}

vioscsi->vm_id = vm->vm_params.vmc_params.vcp_id;
vioscsi->irq = pci_get_dev_irq(vioscsi->pci_id);

/* vioscsi uses 3 virtqueues. */
for (i = 0; i < 3; i++) {
hva = hvaddr_mem(vioscsi->vq[i].q_gpa,
    vring_size(VIOSCSI_QUEUE_SIZE128));
if (hva == NULL((void *)0))
	fatal("failed to restore vioscsi virtqueue");
vioscsi->vq[i].q_hva = hva;
}

return (0);
1046}

1048int
1049virtio_restore(int fd, struct vmd_vm *vm, int child_cdrom,
  int child_disks[][VM_MAX_BASE_PER_DISK4], int *child_taps)
1051{
struct virtio_dev *dev;
int ret;

SLIST_INIT(&virtio_devs){ ((&virtio_devs)->slh_first) = ((void *)0); };

if ((ret = viornd_restore(fd, vm)) == -1)
return (ret);

if ((ret = vioblk_restore(fd, vm, child_disks)) == -1)
return (ret);

if ((ret = vioscsi_restore(fd, vm, child_cdrom)) == -1)
return (ret);

if ((ret = vionet_restore(fd, vm, child_taps)) == -1)
return (ret);

if ((ret = vmmci_restore(fd, vm->vm_params.vmc_params.vcp_id)) == -1)
return (ret);

SLIST_FOREACH(dev, &virtio_devs, dev_next)for((dev) = ((&virtio_devs)->slh_first); (dev) != ((void
 *)0); (dev) = ((dev)->dev_next.sle_next)) {
if (virtio_dev_launch(vm, dev) != 0)
	fatalx("%s: failed to restore virtio dev", __func__);
}

return (0);
1078}

1080int
1081viornd_dump(int fd)
1082{
log_debug("%s: sending viornd", __func__);

viornd.vq[0].q_hva = NULL((void *)0);

if (atomicio(vwrite(ssize_t (*)(int, void *, size_t))write, fd, &viornd, sizeof(viornd)) != sizeof(viornd)) {
log_warnx("%s: error writing viornd to fd", __func__);
return (-1);
}
return (0);
1092}

1094int
1095vmmci_dump(int fd)
1096{
log_debug("%s: sending vmmci", __func__);

if (atomicio(vwrite(ssize_t (*)(int, void *, size_t))write, fd, &vmmci, sizeof(vmmci)) != sizeof(vmmci)) {
log_warnx("%s: error writing vmmci to fd", __func__);
return (-1);
}
return (0);
1104}

1106int
1107vionet_dump(int fd)
1108{
struct virtio_dev	*dev, temp;
struct viodev_msg	 msg;
struct imsg		 imsg;
struct imsgbuf		*ibuf = NULL((void *)0);
size_t			 sz;
int			 ret;

log_debug("%s: dumping vionet", __func__);

SLIST_FOREACH(dev, &virtio_devs, dev_next)for((dev) = ((&virtio_devs)->slh_first); (dev) != ((void
 *)0); (dev) = ((dev)->dev_next.sle_next)) {
if (dev->dev_type != VMD_DEVTYPE_NET'n')
	continue;

memset(&msg, 0, sizeof(msg));
memset(&imsg, 0, sizeof(imsg));

ibuf = &dev->sync_iev.ibuf;
msg.type = VIODEV_MSG_DUMP6;

ret = imsg_compose(ibuf, IMSG_DEVOP_MSG, 0, 0, -1, &msg,
    sizeof(msg));
if (ret == -1) {
	log_warnx("%s: failed requesting dump of vionet[%d]",
	    __func__, dev->vionet.idx);
	return (-1);
}
if (imsg_flush(ibuf) == -1) {
	log_warnx("%s: imsg_flush", __func__);
	return (-1);
}

sz = atomicio(read, dev->sync_fd, &temp, sizeof(temp));
if (sz != sizeof(temp)) {
	log_warnx("%s: failed to dump vionet[%d]", __func__,
	    dev->vionet.idx);
	return (-1);
}

/* Clear volatile state. Will reinitialize on restore. */
temp.vionet.vq[RXQ0].q_hva = NULL((void *)0);
temp.vionet.vq[TXQ1].q_hva = NULL((void *)0);
temp.async_fd = -1;
temp.sync_fd = -1;
memset(&temp.async_iev, 0, sizeof(temp.async_iev));
memset(&temp.sync_iev, 0, sizeof(temp.sync_iev));

if (atomicio(vwrite(ssize_t (*)(int, void *, size_t))write, fd, &temp, sizeof(temp)) != sizeof(temp)) {
	log_warnx("%s: error writing vionet to fd", __func__);
	return (-1);
}
}

return (0);
1162}

1164int
1165vioblk_dump(int fd)
1166{
struct virtio_dev	*dev, temp;
struct viodev_msg	 msg;
struct imsg		 imsg;
struct imsgbuf		*ibuf = NULL((void *)0);
size_t			 sz;
int			 ret;

log_debug("%s: dumping vioblk", __func__);

SLIST_FOREACH(dev, &virtio_devs, dev_next)for((dev) = ((&virtio_devs)->slh_first); (dev) != ((void
 *)0); (dev) = ((dev)->dev_next.sle_next)) {
if (dev->dev_type != VMD_DEVTYPE_DISK'd')
	continue;

memset(&msg, 0, sizeof(msg));
memset(&imsg, 0, sizeof(imsg));

ibuf = &dev->sync_iev.ibuf;
msg.type = VIODEV_MSG_DUMP6;

ret = imsg_compose(ibuf, IMSG_DEVOP_MSG, 0, 0, -1, &msg,
    sizeof(msg));
if (ret == -1) {
	log_warnx("%s: failed requesting dump of vioblk[%d]",
	    __func__, dev->vioblk.idx);
	return (-1);
}
if (imsg_flush(ibuf) == -1) {
	log_warnx("%s: imsg_flush", __func__);
	return (-1);
}


sz = atomicio(read, dev->sync_fd, &temp, sizeof(temp));
if (sz != sizeof(temp)) {
	log_warnx("%s: failed to dump vioblk[%d]", __func__,
	    dev->vioblk.idx);
	return (-1);
}

/* Clear volatile state. Will reinitialize on restore. */
temp.vioblk.vq[0].q_hva = NULL((void *)0);
temp.async_fd = -1;
temp.sync_fd = -1;
memset(&temp.async_iev, 0, sizeof(temp.async_iev));
memset(&temp.sync_iev, 0, sizeof(temp.sync_iev));

if (atomicio(vwrite(ssize_t (*)(int, void *, size_t))write, fd, &temp, sizeof(temp)) != sizeof(temp)) {
	log_warnx("%s: error writing vioblk to fd", __func__);
	return (-1);
}
}

return (0);
1220}

1222int
1223vioscsi_dump(int fd)
1224{
unsigned int i;

if (vioscsi == NULL((void *)0))
return (0);

log_debug("%s: sending vioscsi", __func__);

for (i = 0; i < 3; i++)
vioscsi->vq[i].q_hva = NULL((void *)0);

if (atomicio(vwrite(ssize_t (*)(int, void *, size_t))write, fd, vioscsi, sizeof(struct vioscsi_dev)) !=
   sizeof(struct vioscsi_dev)) {
log_warnx("%s: error writing vioscsi to fd", __func__);
return (-1);
}
return (0);
1241}

1243int
1244virtio_dump(int fd)
1245{
int ret;

if ((ret = viornd_dump(fd)) == -1)
return ret;

if ((ret = vioblk_dump(fd)) == -1)
return ret;

if ((ret = vioscsi_dump(fd)) == -1)
return ret;

if ((ret = vionet_dump(fd)) == -1)
return ret;

if ((ret = vmmci_dump(fd)) == -1)
return ret;

return (0);
1264}

1266void virtio_broadcast_imsg(struct vmd_vm *vm, uint16_t type, void *data,
  uint16_t datalen)
1268{
struct virtio_dev *dev;
int ret;

SLIST_FOREACH(dev, &virtio_devs, dev_next)for((dev) = ((&virtio_devs)->slh_first); (dev) != ((void
 *)0); (dev) = ((dev)->dev_next.sle_next)) {
ret = imsg_compose_event(&dev->async_iev, type, 0, 0, -1, data,
    datalen);
if (ret == -1) {
	log_warnx("%s: failed to broadcast imsg type %u",
	    __func__, type);
}
}

1281}

1283void
1284virtio_stop(struct vmd_vm *vm)
1285{
return virtio_broadcast_imsg(vm, IMSG_VMDOP_PAUSE_VM, NULL((void *)0), 0);
1287}

1289void
1290virtio_start(struct vmd_vm *vm)
1291{
return virtio_broadcast_imsg(vm, IMSG_VMDOP_UNPAUSE_VM, NULL((void *)0), 0);
1293}

1295/*
* Fork+exec a child virtio device. Returns 0 on success.
*/
1298static int
1299virtio_dev_launch(struct vmd_vm *vm, struct virtio_dev *dev)
1300{
char *nargv[12], num[32], vmm_fd[32], vm_name[VM_NAME_MAX64], t[2];
pid_t dev_pid;
int data_fds[VM_MAX_BASE_PER_DISK4], sync_fds[2], async_fds[2], ret = 0;
size_t i, data_fds_sz, sz = 0;
struct viodev_msg msg;
struct imsg imsg;
struct imsgev *iev = &dev->sync_iev;

switch (dev->dev_type) {
case VMD_DEVTYPE_NET'n':
data_fds[0] = dev->vionet.data_fd;
data_fds_sz = 1;
log_debug("%s: launching vionet%d",
    vm->vm_params.vmc_params.vcp_name, dev->vionet.idx);
break;
case VMD_DEVTYPE_DISK'd':
memcpy(&data_fds, dev->vioblk.disk_fd, sizeof(data_fds));
data_fds_sz = dev->vioblk.ndisk_fd;
log_debug("%s: launching vioblk%d",
    vm->vm_params.vmc_params.vcp_name, dev->vioblk.idx);
break;
/* NOTREACHED */
default:
log_warn("%s: invalid device type", __func__);
return (EINVAL22);
}

/* We need two channels: one synchronous (IO reads) and one async. */
if (socketpair(AF_UNIX1, SOCK_STREAM1, PF_UNSPEC0, sync_fds) == -1) {
log_warn("failed to create socketpair");
return (errno(*__errno()));
}
if (socketpair(AF_UNIX1, SOCK_STREAM1, PF_UNSPEC0, async_fds) == -1) {
log_warn("failed to create async socketpair");
return (errno(*__errno()));
}

/* Keep communication channels open after exec. */
if (fcntl(sync_fds[1], F_SETFD2, 0)) {
ret = errno(*__errno());
log_warn("%s: fcntl", __func__);
goto err;
}
if (fcntl(async_fds[1], F_SETFD2, 0)) {
ret = errno(*__errno());
log_warn("%s: fcnt", __func__);
goto err;
}

/* Fork... */
dev_pid = fork();
if (dev_pid == -1) {
ret = errno(*__errno());
log_warn("%s: fork failed", __func__);
goto err;
}

if (dev_pid > 0) {
/* Parent */
close_fd(sync_fds[1]);
close_fd(async_fds[1]);

/* Save the child's pid to help with cleanup. */
dev->dev_pid = dev_pid;

/* Set the channel fds to the child's before sending. */
dev->sync_fd = sync_fds[1];
dev->async_fd = async_fds[1];

/* Close data fds. Only the child device needs them now. */
for (i = 0; i < data_fds_sz; i++)
	close_fd(data_fds[i]);

/* Set our synchronous channel to non-blocking. */
if (fcntl(sync_fds[0], F_SETFL4, O_NONBLOCK0x0004) == -1) {
	ret = errno(*__errno());
	log_warn("%s: fcntl", __func__);
	goto err;
}

/* 1. Send over our configured device. */
log_debug("%s: sending '%c' type device struct", __func__,
	dev->dev_type);
sz = atomicio(vwrite(ssize_t (*)(int, void *, size_t))write, sync_fds[0], dev, sizeof(*dev));
if (sz != sizeof(*dev)) {
	log_warnx("%s: failed to send device", __func__);
	ret = EIO5;
	goto err;
}

/* 2. Send over details on the VM (including memory fds). */
log_debug("%s: sending vm message for '%s'", __func__,
	vm->vm_params.vmc_params.vcp_name);
sz = atomicio(vwrite(ssize_t (*)(int, void *, size_t))write, sync_fds[0], vm, sizeof(*vm));
if (sz != sizeof(*vm)) {
	log_warnx("%s: failed to send vm details", __func__);
	ret = EIO5;
	goto err;
}

/*
 * Initialize our imsg channel to the child device. The initial
 * communication will be synchronous. We expect the child to
 * report itself "ready" to confirm the launch was a success.
 */
imsg_init(&iev->ibuf, sync_fds[0]);
do
	ret = imsg_read(&iev->ibuf);
while (ret == -1 && errno(*__errno()) == EAGAIN35);
if (ret == 0 || ret == -1) {
	log_warnx("%s: failed to receive ready message from "
	    "'%c' type device", __func__, dev->dev_type);
	ret = EIO5;
	goto err;
}
ret = 0;

log_debug("%s: receiving reply", __func__);
if (imsg_get(&iev->ibuf, &imsg) < 1) {
	log_warnx("%s: imsg_get", __func__);
	ret = EIO5;
	goto err;
}
IMSG_SIZE_CHECK(&imsg, &msg)do { if (((&imsg)->hdr.len - sizeof(struct imsg_hdr)) <
 sizeof(*&msg)) fatalx("bad length imsg received (%s)", "&msg"
); } while (0);
memcpy(&msg, imsg.data, sizeof(msg));
imsg_free(&imsg);

if (msg.type != VIODEV_MSG_READY1) {
	log_warnx("%s: expected ready message, got type %d",
	    __func__, msg.type);
	ret = EINVAL22;
	goto err;
}
log_debug("%s: device reports ready via sync channel",
    __func__);

/*
 * Wire in the async event handling, but after reverting back
 * to the parent's fd's.
 */
dev->sync_fd = sync_fds[0];
dev->async_fd = async_fds[0];
vm_device_pipe(dev, virtio_dispatch_dev);
} else {
/* Child */
close_fd(async_fds[0]);
close_fd(sync_fds[0]);

/* Keep data file descriptors open after exec. */
for (i = 0; i < data_fds_sz; i++) {
	log_debug("%s: marking fd %d !close-on-exec", __func__,
	    data_fds[i]);
	if (fcntl(data_fds[i], F_SETFD2, 0)) {
		ret = errno(*__errno());
		log_warn("%s: fcntl", __func__);
		goto err;
	}
}

memset(&nargv, 0, sizeof(nargv));
memset(num, 0, sizeof(num));
snprintf(num, sizeof(num), "%d", sync_fds[1]);
memset(vmm_fd, 0, sizeof(vmm_fd));
snprintf(vmm_fd, sizeof(vmm_fd), "%d", env->vmd_fd);
memset(vm_name, 0, sizeof(vm_name));
snprintf(vm_name, sizeof(vm_name), "%s",
    vm->vm_params.vmc_params.vcp_name);

t[0] = dev->dev_type;
t[1] = '\0';

nargv[0] = env->argv0;
nargv[1] = "-X";
nargv[2] = num;
nargv[3] = "-t";
nargv[4] = t;
nargv[5] = "-i";
nargv[6] = vmm_fd;
nargv[7] = "-p";
nargv[8] = vm_name;
nargv[9] = "-n";
nargv[10] = NULL((void *)0);

if (env->vmd_verbose == 1) {
	nargv[10] = VMD_VERBOSE_1"-v";;
	nargv[11] = NULL((void *)0);
} else if (env->vmd_verbose > 1) {
	nargv[10] = VMD_VERBOSE_2"-vv";;
	nargv[11] = NULL((void *)0);
}

/* Control resumes in vmd.c:main(). */
execvp(nargv[0], nargv);

ret = errno(*__errno());
log_warn("%s: failed to exec device", __func__);
_exit(ret);
/* NOTREACHED */
}

return (ret);

1503err:
close_fd(sync_fds[0]);
close_fd(sync_fds[1]);
close_fd(async_fds[0]);
close_fd(async_fds[1]);
return (ret);
1509}

1511/*
* Initialize an async imsg channel for a virtio device.
*/
1514int
1515vm_device_pipe(struct virtio_dev *dev, void (*cb)(int, short, void *))
1516{
struct imsgev *iev = &dev->async_iev;
int fd = dev->async_fd;

log_debug("%s: initializing '%c' device pipe (fd=%d)", __func__,
   dev->dev_type, fd);

if (fcntl(fd, F_SETFL4, O_NONBLOCK0x0004) == -1) {
log_warn("failed to set nonblocking mode on vm device pipe");
return (-1);
}

imsg_init(&iev->ibuf, fd);
iev->handler = cb;
iev->data = dev;
iev->events = EV_READ0x02;
imsg_event_add(iev);

return (0);
1535}

1537void
1538virtio_dispatch_dev(int fd, short event, void *arg)
1539{
struct virtio_dev	*dev = (struct virtio_dev*)arg;
struct imsgev		*iev = &dev->async_iev;
struct imsgbuf		*ibuf = &iev->ibuf;
struct imsg		 imsg;
struct viodev_msg	 msg;
ssize_t			 n = 0;

if (event & EV_READ0x02) {
if ((n = imsg_read(ibuf)) == -1 && errno(*__errno()) != EAGAIN35)
	fatal("%s: imsg_read", __func__);
if (n == 0) {
	/* this pipe is dead, so remove the event handler */
	log_debug("%s: pipe dead (EV_READ)", __func__);
	event_del(&iev->ev);
	event_loopexit(NULL((void *)0));
	return;
}
}

if (event & EV_WRITE0x04) {
if ((n = msgbuf_write(&ibuf->w)) == -1 && errno(*__errno()) != EAGAIN35)
	fatal("%s: msgbuf_write", __func__);
if (n == 0) {
	/* this pipe is dead, so remove the event handler */
	log_debug("%s: pipe dead (EV_WRITE)", __func__);
	event_del(&iev->ev);
	event_loopexit(NULL((void *)0));
	return;
}
}

for (;;) {
if ((n = imsg_get(ibuf, &imsg)) == -1)
	fatal("%s: imsg_get", __func__);
if (n == 0)
	break;

switch (imsg.hdr.type) {
case IMSG_DEVOP_MSG:
	IMSG_SIZE_CHECK(&imsg, &msg)do { if (((&imsg)->hdr.len - sizeof(struct imsg_hdr)) <
 sizeof(*&msg)) fatalx("bad length imsg received (%s)", "&msg"
); } while (0);
	memcpy(&msg, imsg.data, sizeof(msg));
	handle_dev_msg(&msg, dev);
	break;
default:
	log_warnx("%s: got non devop imsg %d", __func__,
	    imsg.hdr.type);
	break;
}
imsg_free(&imsg);
}
imsg_event_add(iev);
1591}


1594static int
1595handle_dev_msg(struct viodev_msg *msg, struct virtio_dev *gdev)
1596{
uint32_t vm_id = gdev->vm_id;
int irq = gdev->irq;

switch (msg->type) {
case VIODEV_MSG_KICK3:
if (msg->state == INTR_STATE_ASSERT1)
	vcpu_assert_pic_irq(vm_id, msg->vcpu, irq);
else if (msg->state == INTR_STATE_DEASSERT-1)
	vcpu_deassert_pic_irq(vm_id, msg->vcpu, irq);
break;
case VIODEV_MSG_READY1:
log_debug("%s: device reports ready", __func__);
break;
case VIODEV_MSG_ERROR2:
log_warnx("%s: device reported error", __func__);
break;
case VIODEV_MSG_INVALID0:
case VIODEV_MSG_IO_READ4:
case VIODEV_MSG_IO_WRITE5:
/* FALLTHROUGH */
default:
log_warnx("%s: unsupported device message type %d", __func__,
    msg->type);
return (1);
}

return (0);
1624};

1626/*
* Called by the VM process while processing IO from the VCPU thread.
*
* N.b. Since the VCPU thread calls this function, we cannot mutate the event
* system. All ipc messages must be sent manually and cannot be queued for
* the event loop to push them. (We need to perform a synchronous read, so
* this isn't really a big deal.)
*/
1634int
1635virtio_pci_io(int dir, uint16_t reg, uint32_t *data, uint8_t *intr,
  void *cookie, uint8_t sz)
1637{
struct virtio_dev *dev = (struct virtio_dev *)cookie;
struct imsgbuf *ibuf = &dev->sync_iev.ibuf;
struct imsg imsg;
struct viodev_msg msg;
ssize_t n;
int ret = 0;

memset(&msg, 0, sizeof(msg));
msg.reg = reg;
msg.io_sz = sz;

if (dir == 0) {
msg.type = VIODEV_MSG_IO_WRITE5;
msg.data = *data;
msg.data_valid = 1;
} else
msg.type = VIODEV_MSG_IO_READ4;

if (msg.type == VIODEV_MSG_IO_WRITE5) {
/*
 * Write request. No reply expected.
 */
ret = imsg_compose(ibuf, IMSG_DEVOP_MSG, 0, 0, -1, &msg,
    sizeof(msg));
if (ret == -1) {
	log_warn("%s: failed to send async io event to virtio"
	    " device", __func__);
	return (ret);
}
if (imsg_flush(ibuf) == -1) {
	log_warnx("%s: imsg_flush (write)", __func__);
	return (-1);
}
} else {
/*
 * Read request. Requires waiting for a reply.
 */
ret = imsg_compose(ibuf, IMSG_DEVOP_MSG, 0, 0, -1, &msg,
    sizeof(msg));
if (ret == -1) {
	log_warnx("%s: failed to send sync io event to virtio"
	    " device", __func__);
	return (ret);
}
if (imsg_flush(ibuf) == -1) {
	log_warnx("%s: imsg_flush (read)", __func__);
	return (-1);
}

/* Read our reply. */
do
	n = imsg_read(ibuf);
while (n == -1 && errno(*__errno()) == EAGAIN35);
if (n == 0 || n == -1) {
	log_warn("%s: imsg_read (n=%ld)", __func__, n);
	return (-1);
}
if ((n = imsg_get(ibuf, &imsg)) == -1) {
	log_warn("%s: imsg_get (n=%ld)", __func__, n);
	return (-1);
}
if (n == 0) {
	log_warnx("%s: invalid imsg", __func__);
	return (-1);
}

IMSG_SIZE_CHECK(&imsg, &msg)do { if (((&imsg)->hdr.len - sizeof(struct imsg_hdr)) <
 sizeof(*&msg)) fatalx("bad length imsg received (%s)", "&msg"
); } while (0);
memcpy(&msg, imsg.data, sizeof(msg));
imsg_free(&imsg);

if (msg.type == VIODEV_MSG_IO_READ4 && msg.data_valid) {
1709#if DEBUG
	log_debug("%s: got sync read response (reg=%s)",
	    __func__, virtio_reg_name(msg.reg));
1712#endif /* DEBUG */
	*data = msg.data;
	/*
	 * It's possible we're asked to {de,}assert after the
	 * device performs a register read.
	 */
	if (msg.state == INTR_STATE_ASSERT1)
		vcpu_assert_pic_irq(dev->vm_id, msg.vcpu, msg.irq);
	else if (msg.state == INTR_STATE_DEASSERT-1)
		vcpu_deassert_pic_irq(dev->vm_id, msg.vcpu, msg.irq);
} else {
	log_warnx("%s: expected IO_READ, got %d", __func__,
	    msg.type);
	return (-1);
}
}

return (0);
1730}

1732void
1733virtio_assert_pic_irq(struct virtio_dev *dev, int vcpu)
1734{
struct viodev_msg msg;
int ret;

memset(&msg, 0, sizeof(msg));
msg.irq = dev->irq;
msg.vcpu = vcpu;
msg.type = VIODEV_MSG_KICK3;
msg.state = INTR_STATE_ASSERT1;

ret = imsg_compose_event(&dev->async_iev, IMSG_DEVOP_MSG, 0, 0, -1,
   &msg, sizeof(msg));
if (ret == -1)
log_warnx("%s: failed to assert irq %d", __func__, dev->irq);
1748}

1750void
1751virtio_deassert_pic_irq(struct virtio_dev *dev, int vcpu)
1752{
struct viodev_msg msg;
int ret;

memset(&msg, 0, sizeof(msg));
msg.irq = dev->irq;
msg.vcpu = vcpu;
msg.type = VIODEV_MSG_KICK3;
msg.state = INTR_STATE_DEASSERT-1;

ret = imsg_compose_event(&dev->async_iev, IMSG_DEVOP_MSG, 0, 0, -1,
   &msg, sizeof(msg));
if (ret == -1)
log_warnx("%s: failed to deassert irq %d", __func__, dev->irq);
1766}