vmwpvs(4), a driver for VMware Paravirtual SCSI things in vmware guests.

i got it working on the eurostar while bored, and its at a point where it
will work given a bit more attention rather than just being a way to occupy
my time.

there's some cool use after frees and a lack of hotplug support to work on

ok krw@ jsg@

2 files changed, 986 insertions, 1 deletions

diff --git a/sys/dev/pci/files.pci b/sys/dev/pci/files.pci
index f9d57db5e8f..ec004798b38 100644
--- a/sys/dev/pci/files.pci
+++ b/sys/dev/pci/files.pci
@@ -1,4 +1,4 @@
-#	$OpenBSD: files.pci,v 1.295 2013/08/15 06:54:35 kettenis Exp $
+#	$OpenBSD: files.pci,v 1.296 2013/10/08 07:40:01 dlg Exp $
 #	$NetBSD: files.pci,v 1.20 1996/09/24 17:47:15 christos Exp $
 #
 # Config file and device description for machine-independent PCI code.
@@ -763,6 +763,11 @@ device	vmx: ether, ifnet, ifmedia
 attach	vmx at pci
 file	dev/pci/if_vmx.c		vmx
 
+# VMware Paravirtual SCSI controller
+device	vmwpvs: scsi
+attach	vmwpvs at pci
+file	dev/pci/vmwpvs.c		vmwpvs
+
 # Atheros L2 Ethernet
 device	lii: ether, ifnet, ifmedia, mii
 attach	lii at pci
diff --git a/sys/dev/pci/vmwpvs.c b/sys/dev/pci/vmwpvs.c
new file mode 100644
index 00000000000..bd115821da2
--- /dev/null
+++ b/sys/dev/pci/vmwpvs.c
@@ -0,0 +1,980 @@
+/*	$OpenBSD: vmwpvs.c,v 1.1 2013/10/08 07:40:01 dlg Exp $ */
+
+/*
+ * Copyright (c) 2013 David Gwynne <dlg@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/buf.h>
+#include <sys/device.h>
+#include <sys/ioctl.h>
+#include <sys/malloc.h>
+#include <sys/kernel.h>
+#include <sys/rwlock.h>
+#include <sys/dkio.h>
+
+#include <machine/bus.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+#include <dev/pci/pcidevs.h>
+
+#include <scsi/scsi_all.h>
+#include <scsi/scsi_message.h>
+#include <scsi/scsiconf.h>
+
+/* pushbuttons */
+#define VMWPVS_RING_PAGES	2
+#define VMWPVS_MAXSGL		(MAXPHYS / PAGE_SIZE)
+#define VMWPVS_SENSELEN		roundup(sizeof(struct scsi_sense_data), 16)
+
+/* "chip" definitions */
+
+#define VMWPVS_R_COMMAND	0x0000
+#define VMWPVS_R_COMMAND_DATA	0x0004
+#define VMWPVS_R_COMMAND_STATUS	0x0008
+#define VMWPVS_R_LAST_STS_0	0x0100
+#define VMWPVS_R_LAST_STS_1	0x0104
+#define VMWPVS_R_LAST_STS_2	0x0108
+#define VMWPVS_R_LAST_STS_3	0x010c
+#define VMWPVS_R_INTR_STATUS	0x100c
+#define VMWPVS_R_INTR_MASK	0x2010
+#define VMWPVS_R_KICK_NON_RW_IO	0x3014
+#define VMWPVS_R_DEBUG		0x3018
+#define VMWPVS_R_KICK_RW_IO	0x4018
+
+#define VMWPVS_INTR_CMPL_0	(1 << 0)
+#define VMWPVS_INTR_CMPL_1	(1 << 1)
+#define VMWPVS_INTR_CMPL_MASK	(VMWPVS_INTR_CMPL_0 | VMWPVS_INTR_CMPL_1)
+#define VMWPVS_INTR_MSG_0	(1 << 2)
+#define VMWPVS_INTR_MSG_1	(1 << 3)
+#define VMWPVS_INTR_MSG_MASK	(VMWPVS_INTR_MSG_0 | VMWPVS_INTR_MSG_0)
+#define VMWPVS_INTR_ALL_MASK	(VMWPVS_INTR_CMPL_MASK | VMWPVS_INTR_MSG_MASK)
+
+#define VMWPVS_PAGE_SHIFT	12
+#define VMWPVS_PAGE_SIZE	(1 << VMWPVS_PAGE_SHIFT)
+
+#define VMWPVS_NPG_COMMAND	1
+#define VMWPVS_NPG_INTR_STATUS	1
+#define VMWPVS_NPG_MISC		2
+#define VMWPVS_NPG_KICK_IO	2
+#define VMWPVS_NPG_MSI_X	2
+
+#define VMWPVS_PG_COMMAND	0
+#define VMWPVS_PG_INTR_STATUS	(VMWPVS_PG_COMMAND + \
+				    VMWPVS_NPG_COMMAND * VMWPVS_PAGE_SIZE)
+#define VMWPVS_PG_MISC		(VMWPVS_PG_INTR_STATUS + \
+				    VMWPVS_NPG_INTR_STATUS * VMWPVS_PAGE_SIZE)
+#define VMWPVS_PG_KICK_IO	(VMWPVS_PG_MISC + \
+				    VMWPVS_NPG_MISC * VMWPVS_PAGE_SIZE)
+#define VMWPVS_PG_MSI_X		(VMWPVS_PG_KICK_IO + \
+				    VMWPVS_NPG_KICK_IO * VMWPVS_PAGE_SIZE)
+#define VMMPVS_PG_LEN		(VMWPVS_PG_MSI_X + \
+				    VMWPVS_NPG_MSI_X * VMWPVS_PAGE_SIZE)
+
+struct vmwpvw_ring_state {
+	u_int32_t		req_prod;
+	u_int32_t		req_cons;
+	u_int32_t		req_entries; /* log 2 */
+
+	u_int32_t		cmp_prod;
+	u_int32_t		cmp_cons;
+	u_int32_t		cmp_entries; /* log 2 */
+
+	u_int32_t		__reserved[26];
+
+	u_int32_t		msg_prod;
+	u_int32_t		msg_cons;
+	u_int32_t		msg_entries; /* log 2 */
+} __packed;
+
+struct vmwpvs_ring_req {
+	u_int64_t		context;
+
+	u_int64_t		data_addr;
+	u_int64_t		data_len;
+
+	u_int64_t		sense_addr;
+	u_int32_t		sense_len;
+
+	u_int32_t		flags;
+#define VMWPVS_REQ_SGL			(1 << 0)
+#define VMWPVS_REQ_OOBCDB		(1 << 1)
+#define VMWPVS_REQ_DIR_NONE		(1 << 2)
+#define VMWPVS_REQ_DIR_IN		(1 << 3)
+#define VMWPVS_REQ_DIR_OUT		(1 << 4)
+
+	u_int8_t		cdb[16];
+	u_int8_t		cdblen;
+	u_int8_t		lun[8];
+	u_int8_t		tag;
+	u_int8_t		bus;
+	u_int8_t		target;
+	u_int8_t		vcpu_hint;
+
+	u_int8_t		__reserved[59];
+} __packed;
+#define VMWPVS_REQ_COUNT	((VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE) / \
+				    sizeof(struct vmwpvs_ring_req))
+
+struct vmwpvs_ring_cmp {
+	u_int64_t		context;
+	u_int64_t		data_len;
+	u_int32_t		sense_len;
+	u_int16_t		host_status;
+	u_int16_t		scsi_status;
+	u_int32_t		__reserved[2];
+} __packed;
+#define VMWPVS_CMP_COUNT	((VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE) / \
+				    sizeof(struct vmwpvs_ring_cmp))
+
+struct vmwpvs_sge {
+	u_int64_t		addr;
+	u_int32_t		len;
+	u_int32_t		flags;
+} __packed;
+
+struct vmwpvs_cfg_cmd {
+	u_int64_t		cmp_addr;
+	u_int32_t		pg_addr;
+	u_int32_t		pg_addr_type;
+	u_int32_t		pg_num;
+	u_int32_t		__reserved;
+} __packed;
+
+#define VMWPVS_MAX_RING_PAGES	32
+struct vmwpvs_setup_rings_cmd {
+	u_int32_t		req_pages;
+	u_int32_t		cmp_pages;
+	u_int64_t		state_ppn;
+	u_int64_t		req_page_ppn[VMWPVS_MAX_RING_PAGES];
+	u_int64_t		cmp_page_ppn[VMWPVS_MAX_RING_PAGES];
+} __packed;
+
+#define VMWPVS_CMD_FIRST		0
+#define VMWPVS_CMD_ADAPTER_RESET	1
+#define VMWPVS_CMD_ISSUE_SCSI		2
+#define VMWPVS_CMD_SETUP_RINGS		3
+#define VMWPVS_CMD_RESET_BUS		4
+#define VMWPVS_CMD_RESET_DEVICE		5
+#define VMWPVS_CMD_ABORT_CMD		6
+#define VMWPVS_CMD_CONFIG		7
+#define VMWPVS_CMD_SETUP_MSG_RING	8
+#define VMWPVS_CMD_DEVICE_UNPLUG	9
+#define VMWPVS_CMD_LAST			10
+
+#define VMWPVS_CFGPG_CONTROLLER		0x1958
+#define VMWPVS_CFGPG_PHY		0x1959
+#define VMWPVS_CFGPG_DEVICE		0x195a
+
+#define VMWPVS_CFGPGADDR_CONTROLLER	0x2120
+#define VMWPVS_CFGPGADDR_TARGET		0x2121
+#define VMWPVS_CFGPGADDR_PHY		0x2122
+
+struct vmwpvs_cfg_pg_header {
+	u_int32_t		pg_num;
+	u_int16_t		num_dwords;
+	u_int16_t		host_status;
+	u_int16_t		scsi_status;
+	u_int16_t		__reserved[3];
+} __packed;
+
+#define VMWPVS_HOST_STATUS_SUCCESS	0x00
+#define VMWPVS_HOST_STATUS_LINKED_CMD_COMPLETED 0x0a
+#define VMWPVS_HOST_STATUS_LINKED_CMD_COMPLETED_WITH_FLAG 0x0b
+#define VMWPVS_HOST_STATUS_UNDERRUN	0x0c
+#define VMWPVS_HOST_STATUS_SELTIMEOUT	0x11
+#define VMWPVS_HOST_STATUS_DATARUN	0x12
+#define VMWPVS_HOST_STATUS_BUSFREE	0x13
+#define VMWPVS_HOST_STATUS_INVPHASE	0x14
+#define VMWPVS_HOST_STATUS_LUNMISMATCH	0x17
+#define VMWPVS_HOST_STATUS_INVPARAM	0x1a
+#define VMWPVS_HOST_STATUS_SENSEFAILED	0x1b
+#define VMWPVS_HOST_STATUS_TAGREJECT	0x1c
+#define VMWPVS_HOST_STATUS_BADMSG	0x1d
+#define VMWPVS_HOST_STATUS_HAHARDWARE	0x20
+#define VMWPVS_HOST_STATUS_NORESPONSE	0x21
+#define VMWPVS_HOST_STATUS_SENT_RST	0x22
+#define VMWPVS_HOST_STATUS_RECV_RST	0x23
+#define VMWPVS_HOST_STATUS_DISCONNECT	0x24
+#define VMWPVS_HOST_STATUS_BUS_RESET	0x25
+#define VMWPVS_HOST_STATUS_ABORT_QUEUE	0x26
+#define VMWPVS_HOST_STATUS_HA_SOFTWARE	0x27
+#define VMWPVS_HOST_STATUS_HA_TIMEOUT	0x30
+#define VMWPVS_HOST_STATUS_SCSI_PARITY	0x34
+
+#define VMWPVS_SCSI_STATUS_OK		0x00
+#define VMWPVS_SCSI_STATUS_CHECK	0x02
+
+struct vmwpvs_cfg_pg_controller {
+	struct vmwpvs_cfg_pg_header header;
+
+	u_int64_t		wwnn;
+	u_int16_t		manufacturer[64];
+	u_int16_t		serial_number[64];
+	u_int16_t		oprom_version[32];
+	u_int16_t		hardware_version[32];
+	u_int16_t		firmware_version[32];
+	u_int32_t		num_phys;
+	u_int8_t		use_consec_phy_wwns;
+	u_int8_t		__reserved[3];
+} __packed;
+
+/* driver stuff */
+
+struct vmwpvs_dmamem {
+	bus_dmamap_t		dm_map;
+	bus_dma_segment_t	dm_seg;
+	size_t			dm_size;
+	caddr_t			dm_kva;
+};
+#define VMWPVS_DMA_MAP(_dm)	(_dm)->dm_map
+#define VMWPVS_DMA_DVA(_dm)	(_dm)->dm_map->dm_segs[0].ds_addr
+#define VMWPVS_DMA_KVA(_dm)	(void *)(_dm)->dm_kva
+
+struct vmwpvs_sgl {
+	struct vmwpvs_sge	list[VMWPVS_MAXSGL];
+} __packed;
+
+struct vmwpvs_ccb {
+	SLIST_ENTRY(vmwpvs_ccb)	ccb_entry;
+
+	bus_dmamap_t		ccb_dmamap;
+	struct scsi_xfer	*ccb_xs;
+	u_int64_t		ccb_ctx;
+
+	struct vmwpvs_sgl	*ccb_sgl;
+	bus_addr_t		ccb_sgl_offset;
+
+	void			*ccb_sense;
+	bus_addr_t		ccb_sense_offset;
+};
+SLIST_HEAD(vmwpvs_ccb_list, vmwpvs_ccb);
+
+struct vmwpvs_softc {
+	struct device		sc_dev;
+
+	pci_chipset_tag_t	sc_pc;
+	pcitag_t		sc_tag;
+
+	bus_space_tag_t		sc_iot;
+	bus_space_handle_t	sc_ioh;
+	bus_size_t		sc_ios;
+	bus_dma_tag_t		sc_dmat;
+
+	struct vmwpvs_dmamem	*sc_ring_state;
+	struct vmwpvs_dmamem	*sc_req_ring;
+	struct vmwpvs_dmamem	*sc_cmp_ring;
+	struct mutex		sc_ring_mtx;
+
+	struct vmwpvs_dmamem	*sc_sgls;
+	struct vmwpvs_dmamem	*sc_sense;
+	struct vmwpvs_ccb	*sc_ccbs;
+	struct vmwpvs_ccb_list	sc_ccb_list;
+	struct mutex		sc_ccb_mtx;
+
+	void			*sc_ih;
+
+	u_int			sc_bus_width;
+
+	struct scsi_link	sc_link;
+	struct scsi_iopool	sc_iopool;
+	struct scsibus_softc	*sc_scsibus;
+};
+#define DEVNAME(_s)		((_s)->sc_dev.dv_xname)
+
+int	vmwpvs_match(struct device *, void *, void *);
+void	vmwpvs_attach(struct device *, struct device *, void *);
+
+int	vmwpvs_intx(void *);
+int	vmwpvs_intr(void *);
+
+#define vmwpvs_read(_s, _r) \
+	bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r))
+#define vmwpvs_write(_s, _r, _v) \
+	bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v))
+#define vmwpvs_barrier(_s, _r, _l, _d) \
+	bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_d))
+
+struct cfattach vmwpvs_ca = {
+	sizeof(struct vmwpvs_softc),
+	vmwpvs_match,
+	vmwpvs_attach,
+	NULL
+};
+
+struct cfdriver vmwpvs_cd = {
+	NULL,
+	"vmwpvs",
+	DV_DULL
+};
+
+void		vmwpvs_scsi_cmd(struct scsi_xfer *);
+
+struct scsi_adapter vmwpvs_switch = {
+	vmwpvs_scsi_cmd,
+	scsi_minphys,
+	NULL,
+	NULL,
+	NULL
+};
+
+#define dwordsof(s)		(sizeof(s) / sizeof(u_int32_t))
+
+void		vmwpvs_ccb_put(void *, void *);
+void *		vmwpvs_ccb_get(void *);
+
+struct vmwpvs_dmamem *
+		vmwpvs_dmamem_alloc(struct vmwpvs_softc *, size_t);
+struct vmwpvs_dmamem *
+		vmwpvs_dmamem_zalloc(struct vmwpvs_softc *, size_t);
+void		vmwpvs_dmamem_free(struct vmwpvs_softc *,
+		    struct vmwpvs_dmamem *);
+
+void		vmwpvs_cmd(struct vmwpvs_softc *, u_int32_t, void *, size_t);
+int		vmwpvs_get_config(struct vmwpvs_softc *);
+void		vmwpvs_setup_rings(struct vmwpvs_softc *);
+
+void		vmwpvs_scsi_cmd_poll(struct vmwpvs_softc *, u_int64_t);
+u_int64_t	vmwpvs_scsi_cmd_done(struct vmwpvs_softc *,
+		    struct vmwpvs_ring_cmp *);
+
+int
+vmwpvs_match(struct device *parent, void *match, void *aux)
+{
+	struct pci_attach_args *pa = aux;
+
+	if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_VMWARE &&
+	    PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_VMWARE_PVSCSI)
+		return (1);
+
+	return (0);
+}
+
+void
+vmwpvs_attach(struct device *parent, struct device *self, void *aux)
+{
+	struct vmwpvs_softc *sc = (struct vmwpvs_softc *)self;
+	struct pci_attach_args *pa = aux;
+	struct scsibus_attach_args saa;
+	pcireg_t memtype;
+	u_int i, r;
+	int (*isr)(void *) = vmwpvs_intx;
+	pci_intr_handle_t ih;
+
+	struct vmwpvs_ccb *ccb;
+	struct vmwpvs_sgl *sgls;
+	u_int8_t *sense;
+
+	sc->sc_pc = pa->pa_pc;
+	sc->sc_tag = pa->pa_tag;
+	sc->sc_dmat = pa->pa_dmat;
+
+	sc->sc_bus_width = 16;
+	mtx_init(&sc->sc_ring_mtx, IPL_BIO);
+	mtx_init(&sc->sc_ccb_mtx, IPL_BIO);
+	SLIST_INIT(&sc->sc_ccb_list);
+
+	for (r = PCI_MAPREG_START; r < PCI_MAPREG_END; r += sizeof(memtype)) {
+		memtype = pci_mapreg_type(sc->sc_pc, sc->sc_tag, r);
+		if ((memtype & PCI_MAPREG_TYPE_MASK) == PCI_MAPREG_TYPE_MEM)
+			break;
+	}
+	if (r >= PCI_MAPREG_END) {
+		printf(": unable to locate registers\n");
+		return;
+	}
+
+	if (pci_mapreg_map(pa, r, memtype, 0, &sc->sc_iot, &sc->sc_ioh,
+	    NULL, &sc->sc_ios, VMMPVS_PG_LEN) != 0) {
+		printf(": unable to map registers\n");
+		return;
+	}
+
+	/* hook up the interrupt */
+	vmwpvs_write(sc, VMWPVS_R_INTR_MASK, 0);
+
+	if (pci_intr_map_msi(pa, &ih) == 0)
+		isr = vmwpvs_intr;
+	else if (pci_intr_map(pa, &ih) != 0) {
+		printf(": unable to map interrupt\n");
+		goto unmap;
+	}
+
+	printf(": %s\n", pci_intr_string(sc->sc_pc, ih));
+
+	if (vmwpvs_get_config(sc) != 0) {
+		printf("%s: get configuration failed\n", DEVNAME(sc));
+		goto unmap;
+	}
+
+	sc->sc_ring_state = vmwpvs_dmamem_zalloc(sc, VMWPVS_PAGE_SIZE);
+	if (sc->sc_ring_state == NULL) {
+		printf("%s: unable to allocate ring state\n", DEVNAME(sc));
+		goto unmap;
+	}
+
+	sc->sc_req_ring = vmwpvs_dmamem_zalloc(sc,
+	    VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE);
+	if (sc->sc_req_ring == NULL) {
+		printf("%s: unable to allocate req ring\n", DEVNAME(sc));
+		goto free_ring_state;
+	}
+
+	sc->sc_cmp_ring = vmwpvs_dmamem_zalloc(sc,
+	    VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE);
+	if (sc->sc_cmp_ring == NULL) {
+		printf("%s: unable to allocate cmp ring\n", DEVNAME(sc));
+		goto free_req_ring;
+	}
+
+	r = (VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE) /
+	    sizeof(struct vmwpvs_ring_req);
+
+	sc->sc_sgls = vmwpvs_dmamem_alloc(sc, r * sizeof(struct vmwpvs_sgl));
+	if (sc->sc_sgls == NULL) {
+		printf("%s: unable to allocate sgls\n", DEVNAME(sc));
+		goto free_cmp_ring;
+	}
+
+	sc->sc_sense = vmwpvs_dmamem_alloc(sc, r * VMWPVS_SENSELEN);
+	if (sc->sc_sense == NULL) {
+		printf("%s: unable to allocate sense data\n", DEVNAME(sc));
+		goto free_sgl;
+	}
+
+	sc->sc_ccbs = malloc(sizeof(struct vmwpvs_ccb) * r, M_DEVBUF, M_WAITOK);
+	/* cant fail */
+
+	sgls = VMWPVS_DMA_KVA(sc->sc_sgls);
+	sense = VMWPVS_DMA_KVA(sc->sc_sense);
+	for (i = 0; i < r; i++) {
+		ccb = &sc->sc_ccbs[i];
+
+		if (bus_dmamap_create(sc->sc_dmat, MAXPHYS,
+		    VMWPVS_MAXSGL, MAXPHYS, 0,
+		    BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
+		    &ccb->ccb_dmamap) != 0) {
+			printf("%s: unable to create ccb map\n", DEVNAME(sc));
+			goto free_ccbs;
+		}
+
+		ccb->ccb_ctx = 0xdeadbeef00000000ULL | (u_int64_t)i;
+
+		ccb->ccb_sgl_offset = i * sizeof(*sgls);
+		ccb->ccb_sgl = &sgls[i];
+
+		ccb->ccb_sense_offset = i * VMWPVS_SENSELEN;
+		ccb->ccb_sense = sense + ccb->ccb_sense_offset;
+
+		vmwpvs_ccb_put(sc, ccb);
+	}
+
+
+	sc->sc_ih = pci_intr_establish(sc->sc_pc, ih, IPL_BIO,
+	    isr, sc, DEVNAME(sc));
+	if (sc->sc_ih == NULL)
+		goto unmap;
+
+	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_cmp_ring), 0,
+	    VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD);
+	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_req_ring), 0,
+	    VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREWRITE);
+	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0,
+	    VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+
+	vmwpvs_setup_rings(sc);
+	vmwpvs_write(sc, VMWPVS_R_INTR_MASK, VMWPVS_INTR_CMPL_MASK);
+
+	scsi_iopool_init(&sc->sc_iopool, sc, vmwpvs_ccb_get, vmwpvs_ccb_put);
+
+	sc->sc_link.adapter = &vmwpvs_switch;
+	sc->sc_link.adapter_softc = sc;
+	sc->sc_link.adapter_target = -1;
+	sc->sc_link.adapter_buswidth = sc->sc_bus_width;
+	sc->sc_link.openings = 16; /* XXX */
+	sc->sc_link.pool = &sc->sc_iopool;
+
+	bzero(&saa, sizeof(saa));
+	saa.saa_sc_link = &sc->sc_link;
+
+	sc->sc_scsibus = (struct scsibus_softc *)config_found(&sc->sc_dev,
+	    &saa, scsiprint);
+
+	return;
+
+free_ccbs:
+	while ((ccb = vmwpvs_ccb_get(sc)) != NULL)
+		bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
+	free(sc->sc_ccbs, M_DEVBUF);
+/* free_sense: */
+	vmwpvs_dmamem_free(sc, sc->sc_sense);
+free_sgl:
+	vmwpvs_dmamem_free(sc, sc->sc_sgls);
+free_cmp_ring:
+	vmwpvs_dmamem_free(sc, sc->sc_cmp_ring);
+free_req_ring:
+	vmwpvs_dmamem_free(sc, sc->sc_req_ring);
+free_ring_state:
+	vmwpvs_dmamem_free(sc, sc->sc_ring_state);
+unmap:
+	bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_ios);
+	sc->sc_ios = 0;
+}
+
+void
+vmwpvs_setup_rings(struct vmwpvs_softc *sc)
+{
+	struct vmwpvs_setup_rings_cmd cmd;
+	u_int64_t ppn;
+	u_int i;
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.req_pages = VMWPVS_RING_PAGES;
+	cmd.cmp_pages = VMWPVS_RING_PAGES;
+	cmd.state_ppn = VMWPVS_DMA_DVA(sc->sc_ring_state) >> VMWPVS_PAGE_SHIFT;
+
+	ppn = VMWPVS_DMA_DVA(sc->sc_req_ring) >> VMWPVS_PAGE_SHIFT;
+	for (i = 0; i < VMWPVS_RING_PAGES; i++)
+		cmd.req_page_ppn[i] = ppn + i;
+
+	ppn = VMWPVS_DMA_DVA(sc->sc_cmp_ring) >> VMWPVS_PAGE_SHIFT;
+	for (i = 0; i < VMWPVS_RING_PAGES; i++)
+		cmd.cmp_page_ppn[i] = ppn + i;
+
+	vmwpvs_cmd(sc, VMWPVS_CMD_SETUP_RINGS, &cmd, sizeof(cmd));
+}
+
+int
+vmwpvs_get_config(struct vmwpvs_softc *sc)
+{
+	struct vmwpvs_cfg_cmd cmd;
+	struct vmwpvs_dmamem *dm;
+	struct vmwpvs_cfg_pg_controller *pg;
+	struct vmwpvs_cfg_pg_header *hdr;
+	int rv = 0;
+
+	dm = vmwpvs_dmamem_alloc(sc, VMWPVS_PAGE_SIZE);
+	if (dm == NULL)
+		return (ENOMEM);
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.cmp_addr = VMWPVS_DMA_DVA(dm);
+	cmd.pg_addr_type = VMWPVS_CFGPGADDR_CONTROLLER;
+	cmd.pg_num = VMWPVS_CFGPG_CONTROLLER;
+
+	pg = VMWPVS_DMA_KVA(dm);
+	memset(pg, 0, VMWPVS_PAGE_SIZE);
+	hdr = &pg->header;
+	hdr->host_status = VMWPVS_HOST_STATUS_INVPARAM;
+	hdr->scsi_status = VMWPVS_SCSI_STATUS_CHECK;
+
+	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(dm), 0, VMWPVS_PAGE_SIZE,
+	    BUS_DMASYNC_PREREAD);
+	vmwpvs_cmd(sc, VMWPVS_CMD_CONFIG, &cmd, sizeof(cmd));
+	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(dm), 0, VMWPVS_PAGE_SIZE,
+	    BUS_DMASYNC_POSTREAD);
+
+	if (hdr->host_status != VMWPVS_HOST_STATUS_SUCCESS ||
+	    hdr->scsi_status != VMWPVS_SCSI_STATUS_OK) {
+		rv = EIO;
+		goto done;
+	}
+
+	sc->sc_bus_width = pg->num_phys;
+
+done:
+	vmwpvs_dmamem_free(sc, dm);
+
+	return (rv);
+
+}
+
+void
+vmwpvs_cmd(struct vmwpvs_softc *sc, u_int32_t cmd, void *buf, size_t len)
+{
+	u_int32_t *p = buf;
+	u_int i;
+
+	len /= sizeof(*p);
+
+	vmwpvs_write(sc, VMWPVS_R_COMMAND, cmd);
+	for (i = 0; i < len; i++)
+		vmwpvs_write(sc, VMWPVS_R_COMMAND_DATA, p[i]);
+}
+
+int
+vmwpvs_intx(void *xsc)
+{
+	struct vmwpvs_softc *sc = xsc;
+	u_int32_t status;
+
+	status = vmwpvs_read(sc, VMWPVS_R_INTR_STATUS);
+	if ((status & VMWPVS_INTR_ALL_MASK) == 0)
+		return (0);
+
+	vmwpvs_write(sc, VMWPVS_R_INTR_STATUS, status);
+
+	return (vmwpvs_intr(sc));
+}
+
+int
+vmwpvs_intr(void *xsc)
+{
+	struct vmwpvs_softc *sc = xsc;
+	volatile struct vmwpvw_ring_state *s =
+	    VMWPVS_DMA_KVA(sc->sc_ring_state);
+	struct vmwpvs_ring_cmp *ring = VMWPVS_DMA_KVA(sc->sc_cmp_ring);
+	u_int32_t cons, prod;
+
+	mtx_enter(&sc->sc_ring_mtx);
+
+	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0,
+	    VMWPVS_PAGE_SIZE, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+	cons = s->cmp_cons;
+	prod = s->cmp_prod;
+	s->cmp_cons = prod;
+
+	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0,
+	    VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+
+	if (cons != prod) {
+		bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_cmp_ring),
+		    0, VMWPVS_PAGE_SIZE, BUS_DMASYNC_POSTREAD);
+
+		do {
+			vmwpvs_scsi_cmd_done(sc,
+			    &ring[cons++ % VMWPVS_CMP_COUNT]);
+		} while (cons != prod);
+
+		bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_cmp_ring),
+		    0, VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD);
+	}
+
+	mtx_leave(&sc->sc_ring_mtx);
+
+	return (1);
+}
+
+void
+vmwpvs_scsi_cmd(struct scsi_xfer *xs)
+{
+	struct scsi_link *link = xs->sc_link;
+	struct vmwpvs_softc *sc = link->adapter_softc;
+	struct vmwpvs_ccb *ccb = xs->io;
+	bus_dmamap_t dmap = ccb->ccb_dmamap;
+	volatile struct vmwpvw_ring_state *s =
+	    VMWPVS_DMA_KVA(sc->sc_ring_state);
+	struct vmwpvs_ring_req *ring = VMWPVS_DMA_KVA(sc->sc_req_ring), *r;
+	u_int32_t prod;
+	u_int64_t ctx = ccb->ccb_ctx;
+	int error;
+	u_int i;
+
+	ccb->ccb_xs = xs;
+
+	if (xs->datalen > 0) {
+		error = bus_dmamap_load(sc->sc_dmat, dmap,
+		    xs->data, xs->datalen, NULL, (xs->flags & SCSI_NOSLEEP) ?
+		    BUS_DMA_NOWAIT : BUS_DMA_WAITOK);
+		if (error) {
+			xs->error = XS_DRIVER_STUFFUP;
+			scsi_done(xs);
+			return;
+		}
+
+		bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
+		    (xs->flags & SCSI_DATA_IN) ? BUS_DMASYNC_PREREAD :
+		    BUS_DMASYNC_PREWRITE);
+	}
+
+	mtx_enter(&sc->sc_ring_mtx);
+
+	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0,
+	    VMWPVS_PAGE_SIZE, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+
+	prod = s->req_prod;
+	r = &ring[prod];
+
+	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_req_ring),
+	    prod * sizeof(*r), sizeof(*r), BUS_DMASYNC_POSTWRITE);
+
+	memset(r, 0, sizeof(*r));
+	r->context = ctx;
+
+	if (xs->datalen > 0) {
+		r->data_len = xs->datalen;
+		if (dmap->dm_nsegs == 1) {
+			r->data_addr = dmap->dm_segs[0].ds_addr;
+		} else {
+			struct vmwpvs_sge *sgl = ccb->ccb_sgl->list, *sge;
+
+			r->data_addr = VMWPVS_DMA_DVA(sc->sc_sgls) +
+			    ccb->ccb_sgl_offset;
+			r->flags = VMWPVS_REQ_SGL;
+
+			for (i = 0; i < dmap->dm_nsegs; i++) {
+				sge = &sgl[i];
+				sge->addr = dmap->dm_segs[i].ds_addr;
+				sge->len = dmap->dm_segs[i].ds_len;
+				sge->flags = 0;
+			}
+
+			bus_dmamap_sync(sc->sc_dmat,
+			    VMWPVS_DMA_MAP(sc->sc_sgls), ccb->ccb_sgl_offset,
+			    sizeof(*sge) * dmap->dm_nsegs,
+			    BUS_DMASYNC_PREWRITE);
+		}
+	}
+	r->sense_addr = VMWPVS_DMA_DVA(sc->sc_sense) + ccb->ccb_sense_offset;
+	r->sense_len = sizeof(xs->sense);
+
+	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_req_ring), 0,
+	    VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE, BUS_DMASYNC_POSTWRITE);
+
+	switch (xs->flags & (SCSI_DATA_IN | SCSI_DATA_OUT)) {
+	case SCSI_DATA_IN:
+		r->flags |= VMWPVS_REQ_DIR_IN;
+		break;
+	case SCSI_DATA_OUT:
+		r->flags |= VMWPVS_REQ_DIR_OUT;
+		break;
+	default:
+		r->flags |= VMWPVS_REQ_DIR_NONE;
+		break;
+	}
+
+	memcpy(r->cdb, xs->cmd, xs->cmdlen);
+	r->cdblen = xs->cmdlen;
+	r->lun[1] = link->lun; /* ugly :( */
+	r->tag = MSG_SIMPLE_Q_TAG;
+	r->bus = 0;
+	r->target = link->target;
+	r->vcpu_hint = 0;
+
+	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_req_ring), 0,
+	    VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREWRITE);
+
+	s->req_prod = prod + 1;
+
+	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0,
+	    VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+
+	vmwpvs_write(sc, xs->bp == NULL ?
+	    VMWPVS_R_KICK_NON_RW_IO : VMWPVS_R_KICK_RW_IO, 0);
+
+	if (ISSET(xs->flags, SCSI_POLL))
+		vmwpvs_scsi_cmd_poll(sc, ctx);
+
+	mtx_leave(&sc->sc_ring_mtx);
+}
+
+void
+vmwpvs_scsi_cmd_poll(struct vmwpvs_softc *sc, u_int64_t req)
+{
+	volatile struct vmwpvw_ring_state *s =
+	    VMWPVS_DMA_KVA(sc->sc_ring_state);
+	struct vmwpvs_ring_cmp *ring = VMWPVS_DMA_KVA(sc->sc_cmp_ring);
+	u_int32_t prod, cons;
+	u_int64_t cmp;
+
+	do {
+		bus_dmamap_sync(sc->sc_dmat,
+		    VMWPVS_DMA_MAP(sc->sc_ring_state), 0, VMWPVS_PAGE_SIZE,
+		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+
+		cons = s->cmp_cons;
+		prod = s->cmp_prod;
+
+		if (cons != prod)
+			s->cmp_cons = cons + 1;
+
+		bus_dmamap_sync(sc->sc_dmat,
+		    VMWPVS_DMA_MAP(sc->sc_ring_state), 0, VMWPVS_PAGE_SIZE,
+		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+
+		if (cons == prod) {
+			delay(1000);
+			continue;
+		}
+
+		bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_cmp_ring),
+		    0, VMWPVS_PAGE_SIZE * VMWPVS_RING_PAGES,
+		    BUS_DMASYNC_POSTREAD);
+		cmp = vmwpvs_scsi_cmd_done(sc, &ring[cons % VMWPVS_CMP_COUNT]);
+		bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_cmp_ring),
+		    0, VMWPVS_PAGE_SIZE * VMWPVS_RING_PAGES,
+		    BUS_DMASYNC_PREREAD);
+	} while (cmp != req);
+}
+
+u_int64_t
+vmwpvs_scsi_cmd_done(struct vmwpvs_softc *sc, struct vmwpvs_ring_cmp *c)
+{
+	u_int64_t ctx = c->context;
+	struct vmwpvs_ccb *ccb = &sc->sc_ccbs[ctx & 0xffffffff];
+	bus_dmamap_t dmap = ccb->ccb_dmamap;
+	struct scsi_xfer *xs = ccb->ccb_xs;
+
+	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_sense),
+	    ccb->ccb_sense_offset, sizeof(xs->sense), BUS_DMASYNC_POSTREAD);
+
+	if (xs->datalen > 0) {
+		if (dmap->dm_nsegs > 1) {
+			bus_dmamap_sync(sc->sc_dmat,
+			    VMWPVS_DMA_MAP(sc->sc_sgls), ccb->ccb_sgl_offset,
+			    sizeof(struct vmwpvs_sge) * dmap->dm_nsegs,
+			    BUS_DMASYNC_POSTWRITE);
+		}
+
+		bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
+		    (xs->flags & SCSI_DATA_IN) ? BUS_DMASYNC_POSTREAD :
+		    BUS_DMASYNC_POSTWRITE);
+
+		bus_dmamap_unload(sc->sc_dmat, dmap);
+	}
+	printf("%s: %s:%d c:0x%02x h:0x%x s:0x%x\n", DEVNAME(sc),
+	    __FUNCTION__, __LINE__, xs->cmd->opcode,
+	    c->host_status, c->scsi_status);
+
+	xs->status = c->scsi_status;
+	switch (c->host_status) {
+	case VMWPVS_HOST_STATUS_SUCCESS:
+	case VMWPVS_HOST_STATUS_LINKED_CMD_COMPLETED:
+	case VMWPVS_HOST_STATUS_LINKED_CMD_COMPLETED_WITH_FLAG:
+		if (c->scsi_status == VMWPVS_SCSI_STATUS_CHECK) {
+			memcpy(&xs->sense, ccb->ccb_sense, sizeof(xs->sense));
+			xs->error = XS_SENSE;
+		} else
+			xs->error = XS_NOERROR;
+		break;
+
+	case VMWPVS_HOST_STATUS_UNDERRUN:
+	case VMWPVS_HOST_STATUS_DATARUN:
+		xs->resid = xs->datalen - c->data_len;
+		xs->error = XS_NOERROR;
+		break;
+
+	case VMWPVS_HOST_STATUS_SELTIMEOUT:
+		xs->error = XS_SELTIMEOUT;
+		break;
+
+	default:
+		printf("%s: %s:%d h:0x%x s:0x%x\n", DEVNAME(sc),
+		    __FUNCTION__, __LINE__, c->host_status, c->scsi_status);
+		xs->error = XS_DRIVER_STUFFUP;
+		break;
+	}
+
+	scsi_done(xs);
+
+	return (ctx);
+}
+
+
+
+
+void *
+vmwpvs_ccb_get(void *xsc)
+{
+	struct vmwpvs_softc *sc = xsc;
+	struct vmwpvs_ccb *ccb;
+
+	mtx_enter(&sc->sc_ccb_mtx);
+	ccb = SLIST_FIRST(&sc->sc_ccb_list);
+	if (ccb != NULL)
+		SLIST_REMOVE_HEAD(&sc->sc_ccb_list, ccb_entry);
+	mtx_leave(&sc->sc_ccb_mtx);
+
+	return (ccb);
+}
+
+void
+vmwpvs_ccb_put(void *xsc, void *io)
+{
+	struct vmwpvs_softc *sc = xsc;
+	struct vmwpvs_ccb *ccb = io;
+
+	mtx_enter(&sc->sc_ccb_mtx);
+	SLIST_INSERT_HEAD(&sc->sc_ccb_list, ccb, ccb_entry);
+	mtx_leave(&sc->sc_ccb_mtx);
+}
+
+struct vmwpvs_dmamem *
+vmwpvs_dmamem_alloc(struct vmwpvs_softc *sc, size_t size)
+{
+	struct vmwpvs_dmamem *dm;
+	int nsegs;
+
+	dm = malloc(sizeof(*dm), M_DEVBUF, M_NOWAIT | M_ZERO);
+	if (dm == NULL)
+		return (NULL);
+
+	dm->dm_size = size;
+
+	if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
+	    BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &dm->dm_map) != 0)
+		goto dmfree;
+
+	if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &dm->dm_seg,
+	    1, &nsegs, BUS_DMA_NOWAIT | BUS_DMA_ZERO) != 0)
+		goto destroy;
+
+	if (bus_dmamem_map(sc->sc_dmat, &dm->dm_seg, nsegs, size,
+	    &dm->dm_kva, BUS_DMA_NOWAIT) != 0)
+		goto free;
+
+	if (bus_dmamap_load(sc->sc_dmat, dm->dm_map, dm->dm_kva, size,
+	    NULL, BUS_DMA_NOWAIT) != 0)
+		goto unmap;
+
+	return (dm);
+
+unmap:
+	bus_dmamem_unmap(sc->sc_dmat, dm->dm_kva, size);
+free:
+	bus_dmamem_free(sc->sc_dmat, &dm->dm_seg, 1);
+destroy:
+	bus_dmamap_destroy(sc->sc_dmat, dm->dm_map);
+dmfree:
+	free(dm, M_DEVBUF);
+
+	return (NULL);
+}
+
+struct vmwpvs_dmamem *
+vmwpvs_dmamem_zalloc(struct vmwpvs_softc *sc, size_t size)
+{
+	struct vmwpvs_dmamem *dm;
+
+	dm = vmwpvs_dmamem_alloc(sc, size);
+	if (dm == NULL)
+		return (NULL);
+
+	memset(VMWPVS_DMA_KVA(dm), 0, size);
+
+	return (dm);
+}
+
+void
+vmwpvs_dmamem_free(struct vmwpvs_softc *sc, struct vmwpvs_dmamem *dm)
+{
+	bus_dmamap_unload(sc->sc_dmat, dm->dm_map);
+	bus_dmamem_unmap(sc->sc_dmat, dm->dm_kva, dm->dm_size);
+	bus_dmamem_free(sc->sc_dmat, &dm->dm_seg, 1);
+	bus_dmamap_destroy(sc->sc_dmat, dm->dm_map);
+	free(dm, M_DEVBUF);
+}