vmd(8) sometimes attempts page-crossing data copies between the host

and guest. The readpage/writepage ioctls of vmm(4) do not support this
and they return EINVAL on such attempts since recently.

Avoid page-crossing guest memory accesses by changing read_page() and
write_page() into read_mem() and write_mem() that can copy arbitrary
lengths of data between host<->guest without page-crossing accesses.

This also allows us to remove page-wise copy-loops in a few places.

ok mlarkin@

4 files changed, 133 insertions, 148 deletions

diff --git a/usr.sbin/vmd/loadfile_elf.c b/usr.sbin/vmd/loadfile_elf.c
index cd5771aeae8..6f93d50bed4 100644
--- a/usr.sbin/vmd/loadfile_elf.c
+++ b/usr.sbin/vmd/loadfile_elf.c
@@ -1,5 +1,5 @@
 /* $NetBSD: loadfile.c,v 1.10 2000/12/03 02:53:04 tsutsui Exp $ */
-/* $OpenBSD: loadfile_elf.c,v 1.8 2016/01/05 06:55:28 mlarkin Exp $ */
+/* $OpenBSD: loadfile_elf.c,v 1.9 2016/01/16 08:55:40 stefan Exp $ */
 
 /*-
  * Copyright (c) 1997 The NetBSD Foundation, Inc.
@@ -191,7 +191,7 @@ push_gdt(void)
 	setsegment(&sd[1], 0, 0xffffffff, SDT_MEMERA, SEL_KPL, 1, 1);
 	setsegment(&sd[2], 0, 0xffffffff, SDT_MEMRWA, SEL_KPL, 1, 1);
 
-	write_page(GDT_PAGE, gdtpage, PAGE_SIZE, 1);
+	write_mem(GDT_PAGE, gdtpage, PAGE_SIZE, 1);
 }
 
 /*
@@ -302,7 +302,7 @@ push_bootargs(int mem_sz)
 	ba[sz + 2] = (int)sizeof(bios_consdev_t) + 3 * sizeof(int);
 	memcpy(&ba[sz + 3], &consdev, sizeof(bios_consdev_t));
 
-	write_page(BOOTARGS_PAGE, ba, PAGE_SIZE, 1);
+	write_mem(BOOTARGS_PAGE, ba, PAGE_SIZE, 1);
 }
 
 /*
@@ -350,7 +350,7 @@ push_stack(int mem_sz, uint32_t end)
 	stack[--loc] = 0x0;
 	stack[--loc] = 0x0;
 
-	write_page(STACK_PAGE, &stack, PAGE_SIZE, 1);
+	write_mem(STACK_PAGE, &stack, PAGE_SIZE, 1);
 
 	return (1024 - (loc - 1)) * sizeof(uint32_t);
 }
@@ -379,7 +379,7 @@ mread(int fd, uint32_t addr, size_t sz)
 
 	/*
 	 * break up the 'sz' bytes into PAGE_SIZE chunks for use with
-	 * write_page
+	 * write_mem
 	 */
 	ct = 0;
 	rd = 0;
@@ -397,7 +397,7 @@ mread(int fd, uint32_t addr, size_t sz)
 		}
 		rd += ct;
 
-		if (write_page(addr, buf, ct, 1))
+		if (write_mem(addr, buf, ct, 1))
 			return (0);
 
 		addr += ct;
@@ -421,7 +421,7 @@ mread(int fd, uint32_t addr, size_t sz)
 		}
 		rd += ct;
 
-		if (write_page(addr, buf, ct, 1))
+		if (write_mem(addr, buf, ct, 1))
 			return (0);
 	}
 
@@ -449,7 +449,7 @@ marc4random_buf(uint32_t addr, int sz)
 
 	/*
 	 * break up the 'sz' bytes into PAGE_SIZE chunks for use with
-	 * write_page
+	 * write_mem
 	 */
 	ct = 0;
 	if (addr % PAGE_SIZE != 0) {
@@ -458,7 +458,7 @@ marc4random_buf(uint32_t addr, int sz)
 
 		arc4random_buf(buf, ct);
 
-		if (write_page(addr, buf, ct, 1))
+		if (write_mem(addr, buf, ct, 1))
 			return;
 
 		addr += ct;
@@ -473,7 +473,7 @@ marc4random_buf(uint32_t addr, int sz)
 
 		arc4random_buf(buf, ct);
 
-		if (write_page(addr, buf, ct, 1))
+		if (write_mem(addr, buf, ct, 1))
 			return;
 	}
 }
@@ -499,14 +499,14 @@ mbzero(uint32_t addr, int sz)
 
 	/*
 	 * break up the 'sz' bytes into PAGE_SIZE chunks for use with
-	 * write_page
+	 * write_mem
 	 */
 	ct = 0;
 	memset(buf, 0, sizeof(buf));
 	if (addr % PAGE_SIZE != 0) {
 		ct = PAGE_SIZE - (addr % PAGE_SIZE);
 
-		if (write_page(addr, buf, ct, 1))
+		if (write_mem(addr, buf, ct, 1))
 			return;
 
 		addr += ct;
@@ -518,7 +518,7 @@ mbzero(uint32_t addr, int sz)
 		else
 			ct = PAGE_SIZE;
 
-		if (write_page(addr, buf, ct, 1))
+		if (write_mem(addr, buf, ct, 1))
 			return;
 	}
 }
diff --git a/usr.sbin/vmd/virtio.c b/usr.sbin/vmd/virtio.c
index 68e685654b5..afb59c8a846 100644
--- a/usr.sbin/vmd/virtio.c
+++ b/usr.sbin/vmd/virtio.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: virtio.c,v 1.7 2016/01/14 02:46:40 mlarkin Exp $	*/
+/*	$OpenBSD: virtio.c,v 1.8 2016/01/16 08:55:40 stefan Exp $	*/
 
 /*
  * Copyright (c) 2015 Mike Larkin <mlarkin@openbsd.org>
@@ -143,25 +143,11 @@ viornd_update_qa(void)
 	viornd.vq[viornd.cfg.queue_select].qa = viornd.cfg.queue_address;
 }
 
-static int
-push_virtio_ring(char *buf, uint32_t gpa, uint32_t vr_sz)
-{
-	uint32_t i;
-
-	for (i = 0; i < vr_sz; i += VIRTIO_PAGE_SIZE) {
-		if (write_page((uint32_t)gpa + i, buf + i, PAGE_SIZE, 0)) {
-			return (1);
-		}
-	}
-
-	return (0);
-}
-
 int
 viornd_notifyq(void)
 {
 	uint64_t q_gpa;
-	uint32_t vr_sz, i;
+	uint32_t vr_sz;
 	int ret;
 	char *buf, *rnd_data;
 	struct vring_desc *desc;
@@ -186,11 +172,9 @@ viornd_notifyq(void)
 
 	memset(buf, 0, vr_sz);
 
-	for (i = 0; i < vr_sz; i += VIRTIO_PAGE_SIZE) {
-		if (read_page((uint32_t)q_gpa + i, buf + i, PAGE_SIZE, 0)) {
-			free(buf);
-			return (0);
-		}
+	if (read_mem((uint32_t)q_gpa, buf, vr_sz, 0)) {
+		free(buf);
+		return (0);
 	}
 
 	desc = (struct vring_desc *)(buf);
@@ -204,7 +188,7 @@ viornd_notifyq(void)
 
 	if (rnd_data != NULL) {
 		arc4random_buf(rnd_data, desc[avail->ring[avail->idx]].len);
-		if (write_page((uint32_t)(desc[avail->ring[avail->idx]].addr),
+		if (write_mem((uint32_t)(desc[avail->ring[avail->idx]].addr),
 		    rnd_data, desc[avail->ring[avail->idx]].len, 0)) {
 			log_warnx("viornd: can't write random data @ "
 			    "0x%llx",
@@ -219,7 +203,7 @@ viornd_notifyq(void)
 			    desc[avail->ring[avail->idx]].len;
 			used->idx++;
 
-			if (push_virtio_ring(buf, q_gpa, vr_sz)) {
+			if (write_mem(q_gpa, buf, vr_sz, 0)) {
 				log_warnx("viornd: error writing vio ring");
 			}
 		}
@@ -372,7 +356,7 @@ int
 vioblk_notifyq(struct vioblk_dev *dev)
 {
 	uint64_t q_gpa;
-	uint32_t vr_sz, i, j, len; //, osz;
+	uint32_t vr_sz; //, osz;
 	uint16_t idx, cmd_desc_idx, secdata_desc_idx, ds_desc_idx; //, dxx;
 	uint8_t ds;
 	int ret;
@@ -401,13 +385,10 @@ vioblk_notifyq(struct vioblk_dev *dev)
 
 	memset(vr, 0, vr_sz);
 
-	for (i = 0; i < vr_sz; i += VIRTIO_PAGE_SIZE) {
-		if (read_page((uint32_t)q_gpa + i, vr + i, PAGE_SIZE, 0)) {
-			log_warnx("error reading gpa 0x%x",
-			    (uint32_t)q_gpa + i);
-			free(vr);
-			return (0);
-		}
+	if (read_mem((uint32_t)q_gpa, vr, vr_sz, 0)) {
+		log_warnx("error reading gpa 0x%x", (uint32_t)q_gpa);
+		free(vr);
+		return (0);
 	}
 
 	/* Compute offsets in ring of descriptors, avail ring, and used ring */
@@ -437,8 +418,8 @@ vioblk_notifyq(struct vioblk_dev *dev)
 	}
 
 	/* Read command from descriptor ring */
-	if (read_page((uint32_t)cmd_desc->addr, &cmd, cmd_desc->len, 0)) {
-		log_warnx("vioblk: command read_page error @ 0x%llx",
+	if (read_mem((uint32_t)cmd_desc->addr, &cmd, cmd_desc->len, 0)) {
+		log_warnx("vioblk: command read_mem error @ 0x%llx",
 		    cmd_desc->addr);
 		free(vr);
 		return (0);
@@ -473,26 +454,15 @@ vioblk_notifyq(struct vioblk_dev *dev)
 				return (0);
 			}
 
-			j = 0;
-			while (j < secdata_desc->len) {
-				if (secdata_desc->len - j >= PAGE_SIZE)
-					len = PAGE_SIZE;
-				else
-					len = secdata_desc->len - j;
-
-				if (write_page((uint32_t)secdata_desc->addr + j,
-				    secdata + j, len, 0)) {
-					log_warnx("can't write sector "
-					    "data to gpa @ 0x%llx",
-					    secdata_desc->addr);
-					dump_descriptor_chain(desc,
-					    cmd_desc_idx);
-					free(vr);
-					free(secdata);
-					return (0);
-				}
-
-				j += PAGE_SIZE;
+			if (write_mem((uint32_t)secdata_desc->addr, secdata,
+			    secdata_desc->len, 0)) {
+				log_warnx("can't write sector "
+				    "data to gpa @ 0x%llx",
+				    secdata_desc->addr);
+				dump_descriptor_chain(desc, cmd_desc_idx);
+				free(vr);
+				free(secdata);
+				return (0);
 			}
 
 			free(secdata);
@@ -507,7 +477,7 @@ vioblk_notifyq(struct vioblk_dev *dev)
 		ds_desc = secdata_desc;
 
 		ds = VIRTIO_BLK_S_OK;
-		if (write_page((uint32_t)ds_desc->addr,
+		if (write_mem((uint32_t)ds_desc->addr,
 		    &ds, ds_desc->len, 0)) {
 			log_warnx("can't write device status data @ "
 			    "0x%llx", ds_desc->addr);
@@ -526,7 +496,7 @@ vioblk_notifyq(struct vioblk_dev *dev)
 		dev->vq[dev->cfg.queue_notify].last_avail = avail->idx &
 		    VIOBLK_QUEUE_MASK;
 
-		if (push_virtio_ring(vr, q_gpa, vr_sz)) {
+		if (write_mem(q_gpa, vr, vr_sz, 0)) {
 			log_warnx("vioblk: error writing vio ring");
 		}
 		break;
@@ -551,25 +521,15 @@ vioblk_notifyq(struct vioblk_dev *dev)
 
 		secbias = 0;
 		do {
-			j = 0;
-			while (j < secdata_desc->len) {
-				if (secdata_desc->len - j >= PAGE_SIZE)
-					len = PAGE_SIZE;
-				else
-					len = secdata_desc->len - j;
-				if (read_page((uint32_t)secdata_desc->addr + j,
-				    secdata + j, len, 0)) {
-					log_warnx("wr vioblk: can't read "
-					    "sector data @ 0x%llx",
-					    secdata_desc->addr);
-					dump_descriptor_chain(desc,
-					     cmd_desc_idx);
-					free(vr);
-					free(secdata);
-					return (0);
-				}
-
-				j += PAGE_SIZE;
+			if (read_mem((uint32_t)secdata_desc->addr, secdata,
+			    secdata_desc->len, 0)) {
+				log_warnx("wr vioblk: can't read "
+				    "sector data @ 0x%llx",
+				    secdata_desc->addr);
+				dump_descriptor_chain(desc, cmd_desc_idx);
+				free(vr);
+				free(secdata);
+				return (0);
 			}
 
 			if (vioblk_do_write(dev, cmd.sector + secbias,
@@ -593,7 +553,7 @@ vioblk_notifyq(struct vioblk_dev *dev)
 		ds_desc = secdata_desc;
 
 		ds = VIRTIO_BLK_S_OK;
-		if (write_page((uint32_t)ds_desc->addr,
+		if (write_mem((uint32_t)ds_desc->addr,
 		    &ds, ds_desc->len, 0)) {
 			log_warnx("wr vioblk: can't write device status "
 			    "data @ 0x%llx", ds_desc->addr);
@@ -610,7 +570,7 @@ vioblk_notifyq(struct vioblk_dev *dev)
 
 		dev->vq[dev->cfg.queue_notify].last_avail = avail->idx &
 		    VIOBLK_QUEUE_MASK;
-		if (push_virtio_ring(vr, q_gpa, vr_sz))
+		if (write_mem(q_gpa, vr, vr_sz, 0))
 			log_warnx("wr vioblk: error writing vio ring");
 		break;
 	case VIRTIO_BLK_T_FLUSH:
@@ -619,7 +579,7 @@ vioblk_notifyq(struct vioblk_dev *dev)
 		ds_desc = &desc[ds_desc_idx];
 
 		ds = VIRTIO_BLK_S_OK;
-		if (write_page((uint32_t)ds_desc->addr,
+		if (write_mem((uint32_t)ds_desc->addr,
 		    &ds, ds_desc->len, 0)) {
 			log_warnx("fl vioblk: can't write device status "
 			    "data @ 0x%llx", ds_desc->addr);
@@ -636,7 +596,7 @@ vioblk_notifyq(struct vioblk_dev *dev)
 
 		dev->vq[dev->cfg.queue_notify].last_avail = avail->idx &
 		    VIOBLK_QUEUE_MASK;
-		if (push_virtio_ring(vr, q_gpa, vr_sz)) {
+		if (write_mem(q_gpa, vr, vr_sz, 0)) {
 			log_warnx("fl vioblk: error writing vio ring");
 		}
 		break;
@@ -826,7 +786,7 @@ int
 vionet_enq_rx(struct vionet_dev *dev, char *pkt, ssize_t sz, int *spc)
 {
 	uint64_t q_gpa;
-	uint32_t vr_sz, i;
+	uint32_t vr_sz;
 	uint16_t idx, pkt_desc_idx, hdr_desc_idx;
 	int ret;
 	char *vr;
@@ -848,13 +808,10 @@ vionet_enq_rx(struct vionet_dev *dev, char *pkt, ssize_t sz, int *spc)
 
 	memset(vr, 0, vr_sz);
 
-	for (i = 0; i < vr_sz; i += VIRTIO_PAGE_SIZE) {
-		if (read_page((uint32_t)q_gpa + i, vr + i, PAGE_SIZE, 0)) {
-			log_warnx("rx enq: error reading gpa 0x%x",
-			    (uint32_t)q_gpa + i);
-			free(vr);
-			return (0);
-		}
+	if (read_mem((uint32_t)q_gpa, vr, vr_sz, 0)) {
+		log_warnx("rx enq: error reading gpa 0x%x", (uint32_t)q_gpa);
+		free(vr);
+		return (0);
 	}
 
 	/* Compute offsets in ring of descriptors, avail ring, and used ring */
@@ -887,8 +844,8 @@ vionet_enq_rx(struct vionet_dev *dev, char *pkt, ssize_t sz, int *spc)
 	}
 
 	/* Write packet to descriptor ring */
-	if (write_page((uint32_t)pkt_desc->addr, pkt, sz, 0)) {
-		log_warnx("vionet: rx enq packet write_page error @ "
+	if (write_mem((uint32_t)pkt_desc->addr, pkt, sz, 0)) {
+		log_warnx("vionet: rx enq packet write_mem error @ "
 		    "0x%llx", pkt_desc->addr);
 		free(vr);
 		return (0);
@@ -901,7 +858,7 @@ vionet_enq_rx(struct vionet_dev *dev, char *pkt, ssize_t sz, int *spc)
 	used->idx++;
 	dev->vq[0].last_avail = (dev->vq[0].last_avail + 1);
 	*spc = avail->idx - dev->vq[0].last_avail;
-	if (push_virtio_ring(vr, q_gpa, vr_sz)) {
+	if (write_mem(q_gpa, vr, vr_sz, 0)) {
 		log_warnx("vionet: error writing vio ring");
 	}
 
@@ -958,7 +915,7 @@ void
 vionet_notify_rx(struct vionet_dev *dev)
 {
 	uint64_t q_gpa;
-	uint32_t vr_sz, i;
+	uint32_t vr_sz;
 	uint16_t idx, pkt_desc_idx;
 	char *vr;
 	struct vring_desc *desc, *pkt_desc;
@@ -977,13 +934,10 @@ vionet_notify_rx(struct vionet_dev *dev)
 
 	memset(vr, 0, vr_sz);
 
-	for (i = 0; i < vr_sz; i += VIRTIO_PAGE_SIZE) {
-		if (read_page((uint32_t)q_gpa + i, vr + i, PAGE_SIZE, 0)) {
-			log_warnx("error reading gpa 0x%x",
-			    (uint32_t)q_gpa + i);
-			free(vr);
-			return;
-		}
+	if (read_mem((uint32_t)q_gpa, vr, vr_sz, 0)) {
+		log_warnx("error reading gpa 0x%x", (uint32_t)q_gpa);
+		free(vr);
+		return;
 	}
 
 	/* Compute offsets in ring of descriptors, avail ring, and used ring */
@@ -1011,7 +965,7 @@ int
 vionet_notifyq(struct vionet_dev *dev)
 {
 	uint64_t q_gpa;
-	uint32_t vr_sz, i;
+	uint32_t vr_sz;
 	uint16_t idx, pkt_desc_idx, hdr_desc_idx, dxx;
 	size_t pktsz;
 	int ret, num_enq, ofs;
@@ -1041,12 +995,9 @@ vionet_notifyq(struct vionet_dev *dev)
 
 	memset(vr, 0, vr_sz);
 
-	for (i = 0; i < vr_sz; i += VIRTIO_PAGE_SIZE) {
-		if (read_page((uint32_t)q_gpa + i, vr + i, PAGE_SIZE, 0)) {
-			log_warnx("error reading gpa 0x%x",
-			    (uint32_t)q_gpa + i);
-			goto out;
-		}
+	if (read_mem((uint32_t)q_gpa, vr, vr_sz, 0)) {
+		log_warnx("error reading gpa 0x%x", (uint32_t)q_gpa);
+		goto out;
 	}
 
 	/* Compute offsets in ring of descriptors, avail ring, and used ring */
@@ -1105,9 +1056,9 @@ vionet_notifyq(struct vionet_dev *dev)
 			}
 
 			/* Read packet from descriptor ring */
-			if (read_page((uint32_t)pkt_desc->addr, pkt + ofs,
+			if (read_mem((uint32_t)pkt_desc->addr, pkt + ofs,
 			    pkt_desc->len, 0)) {
-				log_warnx("vionet: packet read_page error "
+				log_warnx("vionet: packet read_mem error "
 				    "@ 0x%llx", pkt_desc->addr);
 				goto out;
 			}
@@ -1125,9 +1076,9 @@ vionet_notifyq(struct vionet_dev *dev)
 		}
 
 		/* Read packet from descriptor ring */
-		if (read_page((uint32_t)pkt_desc->addr, pkt + ofs,
+		if (read_mem((uint32_t)pkt_desc->addr, pkt + ofs,
 		    pkt_desc->len, 0)) {
-			log_warnx("vionet: packet read_page error @ "
+			log_warnx("vionet: packet read_mem error @ "
 			    "0x%llx", pkt_desc->addr);
 			goto out;
 		}
@@ -1153,7 +1104,7 @@ vionet_notifyq(struct vionet_dev *dev)
 		    VIONET_QUEUE_MASK;
 	}
 
-	if (push_virtio_ring(vr, q_gpa, vr_sz)) {
+	if (write_mem(q_gpa, vr, vr_sz, 0)) {
 		log_warnx("vionet: tx error writing vio ring");
 	}
 
diff --git a/usr.sbin/vmd/vmd.h b/usr.sbin/vmd/vmd.h
index 0df024978ba..97a2ebdbba1 100644
--- a/usr.sbin/vmd/vmd.h
+++ b/usr.sbin/vmd/vmd.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmd.h,v 1.17 2016/01/02 15:05:21 benno Exp $	*/
+/*	$OpenBSD: vmd.h,v 1.18 2016/01/16 08:55:40 stefan Exp $	*/
 
 /*
  * Copyright (c) 2015 Mike Larkin <mlarkin@openbsd.org>
@@ -114,8 +114,8 @@ char	*get_string(uint8_t *, size_t);
 
 /* vmm.c */
 pid_t	 vmm(struct privsep *, struct privsep_proc *);
-int	 write_page(uint32_t dst, void *buf, uint32_t, int);
-int	 read_page(uint32_t dst, void *buf, uint32_t, int);
+int	 write_mem(uint32_t dst, void *buf, uint32_t, int);
+int	 read_mem(uint32_t dst, void *buf, uint32_t, int);
 int	 opentap(void);
 
 /* control.c */
diff --git a/usr.sbin/vmd/vmm.c b/usr.sbin/vmd/vmm.c
index 5acbdfadf3a..119e0ac37e0 100644
--- a/usr.sbin/vmd/vmm.c
+++ b/usr.sbin/vmd/vmm.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmm.c,v 1.19 2016/01/13 12:55:18 reyk Exp $	*/
+/*	$OpenBSD: vmm.c,v 1.20 2016/01/16 08:55:40 stefan Exp $	*/
 
 /*
  * Copyright (c) 2015 Mike Larkin <mlarkin@openbsd.org>
@@ -1477,17 +1477,16 @@ vcpu_exit(struct vm_run_params *vrp)
 }
 
 /*
- * write_page
+ * write_mem
  *
- * Pushes a page of data from 'buf' into the guest VM's memory
- * at paddr 'dst'.
+ * Pushes data from 'buf' into the guest VM's memory at paddr 'dst'.
  *
  * Parameters:
  *  dst: the destination paddr_t in the guest VM to push into.
  *      If there is no guest paddr mapping at 'dst', a new page will be
  *      faulted in by the VMM (provided 'dst' represents a valid paddr
  *      in the guest's address space)
- *  buf: page of data to push
+ *  buf: data to push
  *  len: size of 'buf'
  *  do_mask: 1 to mask the destination address (for kernel load), 0 to
  *      leave 'dst' unmasked
@@ -1499,8 +1498,10 @@ vcpu_exit(struct vm_run_params *vrp)
  * Note - this function only handles GPAs < 4GB. 
  */
 int
-write_page(uint32_t dst, void *buf, uint32_t len, int do_mask)
+write_mem(uint32_t dst, void *buf, uint32_t len, int do_mask)
 {
+	char *p = buf;
+	uint32_t gpa, n, left;
 	struct vm_writepage_params vwp;
 
 	/*
@@ -1510,21 +1511,36 @@ write_page(uint32_t dst, void *buf, uint32_t len, int do_mask)
 	if (do_mask)
 		dst &= 0xFFFFFFF;
 
-	vwp.vwp_paddr = (paddr_t)dst;
-	vwp.vwp_data = buf;
-	vwp.vwp_vm_id = vm_id;
-	vwp.vwp_len = len;
-	if (ioctl(env->vmd_fd, VMM_IOC_WRITEPAGE, &vwp) < 0) {
-		log_warn("writepage ioctl failed");
-		return (errno);
+	left = len;
+	for (gpa = dst; gpa < dst + len;
+	    gpa = (gpa & ~PAGE_MASK) + PAGE_SIZE) {
+		n = left;
+		if (n > PAGE_SIZE)
+			n = PAGE_SIZE;
+		if (n > (PAGE_SIZE - (gpa & PAGE_MASK)))
+			n = PAGE_SIZE - (gpa & PAGE_MASK);
+
+		vwp.vwp_paddr = (paddr_t)gpa;
+		vwp.vwp_data = p;
+		vwp.vwp_vm_id = vm_id;
+		vwp.vwp_len = n;
+		if (ioctl(env->vmd_fd, VMM_IOC_WRITEPAGE, &vwp) < 0) {
+			log_warn("writepage ioctl failed @ 0x%x: "
+			    "dst = 0x%x, len = 0x%x", gpa, dst, len);
+			return (errno);
+		}
+
+		left -= n;
+		p += n;
 	}
+
 	return (0);
 }
 
 /*
- * read_page
+ * read_mem
  *
- * Reads a page of memory at guest paddr 'src' into 'buf'.
+ * Reads memory at guest paddr 'src' into 'buf'.
  *
  * Parameters:
  *  src: the source paddr_t in the guest VM to read from.
@@ -1540,10 +1556,13 @@ write_page(uint32_t dst, void *buf, uint32_t len, int do_mask)
  * Note - this function only handles GPAs < 4GB.
  */
 int
-read_page(uint32_t src, void *buf, uint32_t len, int do_mask)
+read_mem(uint32_t src, void *buf, uint32_t len, int do_mask)
 {
+	char *p = buf;
+	uint32_t gpa, n, left;
 	struct vm_readpage_params vrp;
 
+
 	/*
 	 * Mask kernel load addresses to avoid uint32_t -> uint64_t cast
 	 * errors
@@ -1551,13 +1570,28 @@ read_page(uint32_t src, void *buf, uint32_t len, int do_mask)
 	if (do_mask)
 		src &= 0xFFFFFFF;
 
-	vrp.vrp_paddr = (paddr_t)src;
-	vrp.vrp_data = buf;
-	vrp.vrp_vm_id = vm_id;
-	vrp.vrp_len = len;
-	if (ioctl(env->vmd_fd, VMM_IOC_READPAGE, &vrp) < 0) {
-		log_warn("readpage ioctl failed");
-		return (errno);
+	left = len;
+	for (gpa = src; gpa < src + len;
+	    gpa = (gpa & ~PAGE_MASK) + PAGE_SIZE) {
+		n = left;
+		if (n > PAGE_SIZE)
+			n = PAGE_SIZE;
+		if (n > (PAGE_SIZE - (gpa & PAGE_MASK)))
+			n = PAGE_SIZE - (gpa & PAGE_MASK);
+
+		vrp.vrp_paddr = (paddr_t)gpa;
+		vrp.vrp_data = p;
+		vrp.vrp_vm_id = vm_id;
+		vrp.vrp_len = n;
+		if (ioctl(env->vmd_fd, VMM_IOC_READPAGE, &vrp) < 0) {
+			log_warn("readpage ioctl failed @ 0x%x: "
+			    "src = 0x%x, len = 0x%x", gpa, src, len);
+			return (errno);
+		}
+
+		left -= n;
+		p += n;
 	}
+
 	return (0);
 }