/* $OpenBSD: vmwpvs.c,v 1.18 2020/06/27 14:29:45 krw Exp $ */ /* * Copyright (c) 2013 David Gwynne * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* pushbuttons */ #define VMWPVS_OPENINGS 64 /* according to the linux driver */ #define VMWPVS_RING_PAGES 2 #define VMWPVS_MAXSGL (MAXPHYS / PAGE_SIZE) #define VMWPVS_SENSELEN roundup(sizeof(struct scsi_sense_data), 16) /* "chip" definitions */ #define VMWPVS_R_COMMAND 0x0000 #define VMWPVS_R_COMMAND_DATA 0x0004 #define VMWPVS_R_COMMAND_STATUS 0x0008 #define VMWPVS_R_LAST_STS_0 0x0100 #define VMWPVS_R_LAST_STS_1 0x0104 #define VMWPVS_R_LAST_STS_2 0x0108 #define VMWPVS_R_LAST_STS_3 0x010c #define VMWPVS_R_INTR_STATUS 0x100c #define VMWPVS_R_INTR_MASK 0x2010 #define VMWPVS_R_KICK_NON_RW_IO 0x3014 #define VMWPVS_R_DEBUG 0x3018 #define VMWPVS_R_KICK_RW_IO 0x4018 #define VMWPVS_INTR_CMPL_0 (1 << 0) #define VMWPVS_INTR_CMPL_1 (1 << 1) #define VMWPVS_INTR_CMPL_MASK (VMWPVS_INTR_CMPL_0 | VMWPVS_INTR_CMPL_1) #define VMWPVS_INTR_MSG_0 (1 << 2) #define VMWPVS_INTR_MSG_1 (1 << 3) #define VMWPVS_INTR_MSG_MASK (VMWPVS_INTR_MSG_0 | VMWPVS_INTR_MSG_1) #define VMWPVS_INTR_ALL_MASK (VMWPVS_INTR_CMPL_MASK | VMWPVS_INTR_MSG_MASK) #define VMWPVS_PAGE_SHIFT 12 #define VMWPVS_PAGE_SIZE (1 << VMWPVS_PAGE_SHIFT) #define VMWPVS_NPG_COMMAND 1 #define VMWPVS_NPG_INTR_STATUS 1 #define VMWPVS_NPG_MISC 2 #define VMWPVS_NPG_KICK_IO 2 #define VMWPVS_NPG_MSI_X 2 #define VMWPVS_PG_COMMAND 0 #define VMWPVS_PG_INTR_STATUS (VMWPVS_PG_COMMAND + \ VMWPVS_NPG_COMMAND * VMWPVS_PAGE_SIZE) #define VMWPVS_PG_MISC (VMWPVS_PG_INTR_STATUS + \ VMWPVS_NPG_INTR_STATUS * VMWPVS_PAGE_SIZE) #define VMWPVS_PG_KICK_IO (VMWPVS_PG_MISC + \ VMWPVS_NPG_MISC * VMWPVS_PAGE_SIZE) #define VMWPVS_PG_MSI_X (VMWPVS_PG_KICK_IO + \ VMWPVS_NPG_KICK_IO * VMWPVS_PAGE_SIZE) #define VMMPVS_PG_LEN (VMWPVS_PG_MSI_X + \ VMWPVS_NPG_MSI_X * VMWPVS_PAGE_SIZE) struct vmwpvw_ring_state { u_int32_t req_prod; u_int32_t req_cons; u_int32_t req_entries; /* log 2 */ u_int32_t cmp_prod; u_int32_t cmp_cons; u_int32_t cmp_entries; /* log 2 */ u_int32_t __reserved[26]; u_int32_t msg_prod; u_int32_t msg_cons; u_int32_t msg_entries; /* log 2 */ } __packed; struct vmwpvs_ring_req { u_int64_t context; u_int64_t data_addr; u_int64_t data_len; u_int64_t sense_addr; u_int32_t sense_len; u_int32_t flags; #define VMWPVS_REQ_SGL (1 << 0) #define VMWPVS_REQ_OOBCDB (1 << 1) #define VMWPVS_REQ_DIR_NONE (1 << 2) #define VMWPVS_REQ_DIR_IN (1 << 3) #define VMWPVS_REQ_DIR_OUT (1 << 4) u_int8_t cdb[16]; u_int8_t cdblen; u_int8_t lun[8]; u_int8_t tag; u_int8_t bus; u_int8_t target; u_int8_t vcpu_hint; u_int8_t __reserved[59]; } __packed; #define VMWPVS_REQ_COUNT ((VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE) / \ sizeof(struct vmwpvs_ring_req)) struct vmwpvs_ring_cmp { u_int64_t context; u_int64_t data_len; u_int32_t sense_len; u_int16_t host_status; u_int16_t scsi_status; u_int32_t __reserved[2]; } __packed; #define VMWPVS_CMP_COUNT ((VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE) / \ sizeof(struct vmwpvs_ring_cmp)) struct vmwpvs_sge { u_int64_t addr; u_int32_t len; u_int32_t flags; } __packed; struct vmwpvs_ring_msg { u_int32_t type; u_int32_t __args[31]; } __packed; #define VMWPVS_MSG_COUNT ((VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE) / \ sizeof(struct vmwpvs_ring_msg)) #define VMWPVS_MSG_T_ADDED 0 #define VMWPVS_MSG_T_REMOVED 1 struct vmwpvs_ring_msg_dev { u_int32_t type; u_int32_t bus; u_int32_t target; u_int8_t lun[8]; u_int32_t __pad[27]; } __packed; struct vmwpvs_cfg_cmd { u_int64_t cmp_addr; u_int32_t pg_addr; u_int32_t pg_addr_type; u_int32_t pg_num; u_int32_t __reserved; } __packed; #define VMWPVS_MAX_RING_PAGES 32 struct vmwpvs_setup_rings_cmd { u_int32_t req_pages; u_int32_t cmp_pages; u_int64_t state_ppn; u_int64_t req_page_ppn[VMWPVS_MAX_RING_PAGES]; u_int64_t cmp_page_ppn[VMWPVS_MAX_RING_PAGES]; } __packed; #define VMWPVS_MAX_MSG_RING_PAGES 16 struct vmwpvs_setup_rings_msg { u_int32_t msg_pages; u_int32_t __reserved; u_int64_t msg_page_ppn[VMWPVS_MAX_MSG_RING_PAGES]; } __packed; #define VMWPVS_CMD_FIRST 0 #define VMWPVS_CMD_ADAPTER_RESET 1 #define VMWPVS_CMD_ISSUE_SCSI 2 #define VMWPVS_CMD_SETUP_RINGS 3 #define VMWPVS_CMD_RESET_BUS 4 #define VMWPVS_CMD_RESET_DEVICE 5 #define VMWPVS_CMD_ABORT_CMD 6 #define VMWPVS_CMD_CONFIG 7 #define VMWPVS_CMD_SETUP_MSG_RING 8 #define VMWPVS_CMD_DEVICE_UNPLUG 9 #define VMWPVS_CMD_LAST 10 #define VMWPVS_CFGPG_CONTROLLER 0x1958 #define VMWPVS_CFGPG_PHY 0x1959 #define VMWPVS_CFGPG_DEVICE 0x195a #define VMWPVS_CFGPGADDR_CONTROLLER 0x2120 #define VMWPVS_CFGPGADDR_TARGET 0x2121 #define VMWPVS_CFGPGADDR_PHY 0x2122 struct vmwpvs_cfg_pg_header { u_int32_t pg_num; u_int16_t num_dwords; u_int16_t host_status; u_int16_t scsi_status; u_int16_t __reserved[3]; } __packed; #define VMWPVS_HOST_STATUS_SUCCESS 0x00 #define VMWPVS_HOST_STATUS_LINKED_CMD_COMPLETED 0x0a #define VMWPVS_HOST_STATUS_LINKED_CMD_COMPLETED_WITH_FLAG 0x0b #define VMWPVS_HOST_STATUS_UNDERRUN 0x0c #define VMWPVS_HOST_STATUS_SELTIMEOUT 0x11 #define VMWPVS_HOST_STATUS_DATARUN 0x12 #define VMWPVS_HOST_STATUS_BUSFREE 0x13 #define VMWPVS_HOST_STATUS_INVPHASE 0x14 #define VMWPVS_HOST_STATUS_LUNMISMATCH 0x17 #define VMWPVS_HOST_STATUS_INVPARAM 0x1a #define VMWPVS_HOST_STATUS_SENSEFAILED 0x1b #define VMWPVS_HOST_STATUS_TAGREJECT 0x1c #define VMWPVS_HOST_STATUS_BADMSG 0x1d #define VMWPVS_HOST_STATUS_HAHARDWARE 0x20 #define VMWPVS_HOST_STATUS_NORESPONSE 0x21 #define VMWPVS_HOST_STATUS_SENT_RST 0x22 #define VMWPVS_HOST_STATUS_RECV_RST 0x23 #define VMWPVS_HOST_STATUS_DISCONNECT 0x24 #define VMWPVS_HOST_STATUS_BUS_RESET 0x25 #define VMWPVS_HOST_STATUS_ABORT_QUEUE 0x26 #define VMWPVS_HOST_STATUS_HA_SOFTWARE 0x27 #define VMWPVS_HOST_STATUS_HA_TIMEOUT 0x30 #define VMWPVS_HOST_STATUS_SCSI_PARITY 0x34 #define VMWPVS_SCSI_STATUS_OK 0x00 #define VMWPVS_SCSI_STATUS_CHECK 0x02 struct vmwpvs_cfg_pg_controller { struct vmwpvs_cfg_pg_header header; u_int64_t wwnn; u_int16_t manufacturer[64]; u_int16_t serial_number[64]; u_int16_t oprom_version[32]; u_int16_t hardware_version[32]; u_int16_t firmware_version[32]; u_int32_t num_phys; u_int8_t use_consec_phy_wwns; u_int8_t __reserved[3]; } __packed; /* driver stuff */ struct vmwpvs_dmamem { bus_dmamap_t dm_map; bus_dma_segment_t dm_seg; size_t dm_size; caddr_t dm_kva; }; #define VMWPVS_DMA_MAP(_dm) (_dm)->dm_map #define VMWPVS_DMA_DVA(_dm) (_dm)->dm_map->dm_segs[0].ds_addr #define VMWPVS_DMA_KVA(_dm) (void *)(_dm)->dm_kva struct vmwpvs_sgl { struct vmwpvs_sge list[VMWPVS_MAXSGL]; } __packed; struct vmwpvs_ccb { SIMPLEQ_ENTRY(vmwpvs_ccb) ccb_entry; bus_dmamap_t ccb_dmamap; struct scsi_xfer *ccb_xs; u_int64_t ccb_ctx; struct vmwpvs_sgl *ccb_sgl; bus_addr_t ccb_sgl_offset; void *ccb_sense; bus_addr_t ccb_sense_offset; }; SIMPLEQ_HEAD(vmwpvs_ccb_list, vmwpvs_ccb); struct vmwpvs_softc { struct device sc_dev; pci_chipset_tag_t sc_pc; pcitag_t sc_tag; bus_space_tag_t sc_iot; bus_space_handle_t sc_ioh; bus_size_t sc_ios; bus_dma_tag_t sc_dmat; struct vmwpvs_dmamem *sc_req_ring; struct vmwpvs_dmamem *sc_cmp_ring; struct vmwpvs_dmamem *sc_msg_ring; struct vmwpvs_dmamem *sc_ring_state; struct mutex sc_ring_mtx; struct vmwpvs_dmamem *sc_sgls; struct vmwpvs_dmamem *sc_sense; struct vmwpvs_ccb *sc_ccbs; struct vmwpvs_ccb_list sc_ccb_list; struct mutex sc_ccb_mtx; void *sc_ih; struct task sc_msg_task; u_int sc_bus_width; struct scsi_link sc_link; struct scsi_iopool sc_iopool; struct scsibus_softc *sc_scsibus; }; #define DEVNAME(_s) ((_s)->sc_dev.dv_xname) int vmwpvs_match(struct device *, void *, void *); void vmwpvs_attach(struct device *, struct device *, void *); int vmwpvs_intx(void *); int vmwpvs_intr(void *); #define vmwpvs_read(_s, _r) \ bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) #define vmwpvs_write(_s, _r, _v) \ bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) #define vmwpvs_barrier(_s, _r, _l, _d) \ bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_d)) struct cfattach vmwpvs_ca = { sizeof(struct vmwpvs_softc), vmwpvs_match, vmwpvs_attach, NULL }; struct cfdriver vmwpvs_cd = { NULL, "vmwpvs", DV_DULL }; void vmwpvs_scsi_cmd(struct scsi_xfer *); struct scsi_adapter vmwpvs_switch = { vmwpvs_scsi_cmd, NULL, NULL, NULL, NULL }; #define dwordsof(s) (sizeof(s) / sizeof(u_int32_t)) void vmwpvs_ccb_put(void *, void *); void * vmwpvs_ccb_get(void *); struct vmwpvs_dmamem * vmwpvs_dmamem_alloc(struct vmwpvs_softc *, size_t); struct vmwpvs_dmamem * vmwpvs_dmamem_zalloc(struct vmwpvs_softc *, size_t); void vmwpvs_dmamem_free(struct vmwpvs_softc *, struct vmwpvs_dmamem *); void vmwpvs_cmd(struct vmwpvs_softc *, u_int32_t, void *, size_t); int vmwpvs_get_config(struct vmwpvs_softc *); void vmwpvs_setup_rings(struct vmwpvs_softc *); void vmwpvs_setup_msg_ring(struct vmwpvs_softc *); void vmwpvs_msg_task(void *); struct vmwpvs_ccb * vmwpvs_scsi_cmd_poll(struct vmwpvs_softc *); struct vmwpvs_ccb * vmwpvs_scsi_cmd_done(struct vmwpvs_softc *, struct vmwpvs_ring_cmp *); int vmwpvs_match(struct device *parent, void *match, void *aux) { struct pci_attach_args *pa = aux; if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_VMWARE && PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_VMWARE_PVSCSI) return (1); return (0); } void vmwpvs_attach(struct device *parent, struct device *self, void *aux) { struct vmwpvs_softc *sc = (struct vmwpvs_softc *)self; struct pci_attach_args *pa = aux; struct scsibus_attach_args saa; pcireg_t memtype; u_int i, r, use_msg; int (*isr)(void *) = vmwpvs_intx; u_int32_t intmask; pci_intr_handle_t ih; struct vmwpvs_ccb *ccb; struct vmwpvs_sgl *sgls; u_int8_t *sense; sc->sc_pc = pa->pa_pc; sc->sc_tag = pa->pa_tag; sc->sc_dmat = pa->pa_dmat; sc->sc_bus_width = 16; mtx_init(&sc->sc_ring_mtx, IPL_BIO); mtx_init(&sc->sc_ccb_mtx, IPL_BIO); task_set(&sc->sc_msg_task, vmwpvs_msg_task, sc); SIMPLEQ_INIT(&sc->sc_ccb_list); for (r = PCI_MAPREG_START; r < PCI_MAPREG_END; r += sizeof(memtype)) { memtype = pci_mapreg_type(sc->sc_pc, sc->sc_tag, r); if ((memtype & PCI_MAPREG_TYPE_MASK) == PCI_MAPREG_TYPE_MEM) break; } if (r >= PCI_MAPREG_END) { printf(": unable to locate registers\n"); return; } if (pci_mapreg_map(pa, r, memtype, 0, &sc->sc_iot, &sc->sc_ioh, NULL, &sc->sc_ios, VMMPVS_PG_LEN) != 0) { printf(": unable to map registers\n"); return; } /* hook up the interrupt */ vmwpvs_write(sc, VMWPVS_R_INTR_MASK, 0); if (pci_intr_map_msi(pa, &ih) == 0) isr = vmwpvs_intr; else if (pci_intr_map(pa, &ih) != 0) { printf(": unable to map interrupt\n"); goto unmap; } printf(": %s\n", pci_intr_string(sc->sc_pc, ih)); /* do we have msg support? */ vmwpvs_write(sc, VMWPVS_R_COMMAND, VMWPVS_CMD_SETUP_MSG_RING); use_msg = (vmwpvs_read(sc, VMWPVS_R_COMMAND_STATUS) != 0xffffffff); if (vmwpvs_get_config(sc) != 0) { printf("%s: get configuration failed\n", DEVNAME(sc)); goto unmap; } sc->sc_ring_state = vmwpvs_dmamem_zalloc(sc, VMWPVS_PAGE_SIZE); if (sc->sc_ring_state == NULL) { printf("%s: unable to allocate ring state\n", DEVNAME(sc)); goto unmap; } sc->sc_req_ring = vmwpvs_dmamem_zalloc(sc, VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE); if (sc->sc_req_ring == NULL) { printf("%s: unable to allocate req ring\n", DEVNAME(sc)); goto free_ring_state; } sc->sc_cmp_ring = vmwpvs_dmamem_zalloc(sc, VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE); if (sc->sc_cmp_ring == NULL) { printf("%s: unable to allocate cmp ring\n", DEVNAME(sc)); goto free_req_ring; } if (use_msg) { sc->sc_msg_ring = vmwpvs_dmamem_zalloc(sc, VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE); if (sc->sc_msg_ring == NULL) { printf("%s: unable to allocate msg ring\n", DEVNAME(sc)); goto free_cmp_ring; } } r = (VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE) / sizeof(struct vmwpvs_ring_req); sc->sc_sgls = vmwpvs_dmamem_alloc(sc, r * sizeof(struct vmwpvs_sgl)); if (sc->sc_sgls == NULL) { printf("%s: unable to allocate sgls\n", DEVNAME(sc)); goto free_msg_ring; } sc->sc_sense = vmwpvs_dmamem_alloc(sc, r * VMWPVS_SENSELEN); if (sc->sc_sense == NULL) { printf("%s: unable to allocate sense data\n", DEVNAME(sc)); goto free_sgl; } sc->sc_ccbs = mallocarray(r, sizeof(struct vmwpvs_ccb), M_DEVBUF, M_WAITOK); /* cant fail */ sgls = VMWPVS_DMA_KVA(sc->sc_sgls); sense = VMWPVS_DMA_KVA(sc->sc_sense); for (i = 0; i < r; i++) { ccb = &sc->sc_ccbs[i]; if (bus_dmamap_create(sc->sc_dmat, MAXPHYS, VMWPVS_MAXSGL, MAXPHYS, 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &ccb->ccb_dmamap) != 0) { printf("%s: unable to create ccb map\n", DEVNAME(sc)); goto free_ccbs; } ccb->ccb_ctx = 0xdeadbeef00000000ULL | (u_int64_t)i; ccb->ccb_sgl_offset = i * sizeof(*sgls); ccb->ccb_sgl = &sgls[i]; ccb->ccb_sense_offset = i * VMWPVS_SENSELEN; ccb->ccb_sense = sense + ccb->ccb_sense_offset; vmwpvs_ccb_put(sc, ccb); } sc->sc_ih = pci_intr_establish(sc->sc_pc, ih, IPL_BIO, isr, sc, DEVNAME(sc)); if (sc->sc_ih == NULL) goto free_msg_ring; bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_cmp_ring), 0, VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD); bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_req_ring), 0, VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREWRITE); if (use_msg) { bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_msg_ring), 0, VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD); } bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0, VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); intmask = VMWPVS_INTR_CMPL_MASK; vmwpvs_setup_rings(sc); if (use_msg) { vmwpvs_setup_msg_ring(sc); intmask |= VMWPVS_INTR_MSG_MASK; } vmwpvs_write(sc, VMWPVS_R_INTR_MASK, intmask); /* controller init is done, lets plug the midlayer in */ scsi_iopool_init(&sc->sc_iopool, sc, vmwpvs_ccb_get, vmwpvs_ccb_put); sc->sc_link.adapter = &vmwpvs_switch; sc->sc_link.adapter_softc = sc; sc->sc_link.adapter_target = SDEV_NO_ADAPTER_TARGET; sc->sc_link.adapter_buswidth = sc->sc_bus_width; sc->sc_link.openings = VMWPVS_OPENINGS; sc->sc_link.pool = &sc->sc_iopool; saa.saa_sc_link = &sc->sc_link; sc->sc_scsibus = (struct scsibus_softc *)config_found(&sc->sc_dev, &saa, scsiprint); return; free_ccbs: while ((ccb = vmwpvs_ccb_get(sc)) != NULL) bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); free(sc->sc_ccbs, M_DEVBUF, r * sizeof(struct vmwpvs_ccb)); /* free_sense: */ vmwpvs_dmamem_free(sc, sc->sc_sense); free_sgl: vmwpvs_dmamem_free(sc, sc->sc_sgls); free_msg_ring: if (use_msg) vmwpvs_dmamem_free(sc, sc->sc_msg_ring); free_cmp_ring: vmwpvs_dmamem_free(sc, sc->sc_cmp_ring); free_req_ring: vmwpvs_dmamem_free(sc, sc->sc_req_ring); free_ring_state: vmwpvs_dmamem_free(sc, sc->sc_ring_state); unmap: bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_ios); sc->sc_ios = 0; } void vmwpvs_setup_rings(struct vmwpvs_softc *sc) { struct vmwpvs_setup_rings_cmd cmd; u_int64_t ppn; u_int i; memset(&cmd, 0, sizeof(cmd)); cmd.req_pages = VMWPVS_RING_PAGES; cmd.cmp_pages = VMWPVS_RING_PAGES; cmd.state_ppn = VMWPVS_DMA_DVA(sc->sc_ring_state) >> VMWPVS_PAGE_SHIFT; ppn = VMWPVS_DMA_DVA(sc->sc_req_ring) >> VMWPVS_PAGE_SHIFT; for (i = 0; i < VMWPVS_RING_PAGES; i++) cmd.req_page_ppn[i] = ppn + i; ppn = VMWPVS_DMA_DVA(sc->sc_cmp_ring) >> VMWPVS_PAGE_SHIFT; for (i = 0; i < VMWPVS_RING_PAGES; i++) cmd.cmp_page_ppn[i] = ppn + i; vmwpvs_cmd(sc, VMWPVS_CMD_SETUP_RINGS, &cmd, sizeof(cmd)); } void vmwpvs_setup_msg_ring(struct vmwpvs_softc *sc) { struct vmwpvs_setup_rings_msg cmd; u_int64_t ppn; u_int i; memset(&cmd, 0, sizeof(cmd)); cmd.msg_pages = VMWPVS_RING_PAGES; ppn = VMWPVS_DMA_DVA(sc->sc_msg_ring) >> VMWPVS_PAGE_SHIFT; for (i = 0; i < VMWPVS_RING_PAGES; i++) cmd.msg_page_ppn[i] = ppn + i; vmwpvs_cmd(sc, VMWPVS_CMD_SETUP_MSG_RING, &cmd, sizeof(cmd)); } int vmwpvs_get_config(struct vmwpvs_softc *sc) { struct vmwpvs_cfg_cmd cmd; struct vmwpvs_dmamem *dm; struct vmwpvs_cfg_pg_controller *pg; struct vmwpvs_cfg_pg_header *hdr; int rv = 0; dm = vmwpvs_dmamem_alloc(sc, VMWPVS_PAGE_SIZE); if (dm == NULL) return (ENOMEM); memset(&cmd, 0, sizeof(cmd)); cmd.cmp_addr = VMWPVS_DMA_DVA(dm); cmd.pg_addr_type = VMWPVS_CFGPGADDR_CONTROLLER; cmd.pg_num = VMWPVS_CFGPG_CONTROLLER; pg = VMWPVS_DMA_KVA(dm); memset(pg, 0, VMWPVS_PAGE_SIZE); hdr = &pg->header; hdr->host_status = VMWPVS_HOST_STATUS_INVPARAM; hdr->scsi_status = VMWPVS_SCSI_STATUS_CHECK; bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(dm), 0, VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD); vmwpvs_cmd(sc, VMWPVS_CMD_CONFIG, &cmd, sizeof(cmd)); bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(dm), 0, VMWPVS_PAGE_SIZE, BUS_DMASYNC_POSTREAD); if (hdr->host_status != VMWPVS_HOST_STATUS_SUCCESS || hdr->scsi_status != VMWPVS_SCSI_STATUS_OK) { rv = EIO; goto done; } sc->sc_bus_width = pg->num_phys; done: vmwpvs_dmamem_free(sc, dm); return (rv); } void vmwpvs_cmd(struct vmwpvs_softc *sc, u_int32_t cmd, void *buf, size_t len) { u_int32_t *p = buf; u_int i; len /= sizeof(*p); vmwpvs_write(sc, VMWPVS_R_COMMAND, cmd); for (i = 0; i < len; i++) vmwpvs_write(sc, VMWPVS_R_COMMAND_DATA, p[i]); } int vmwpvs_intx(void *xsc) { struct vmwpvs_softc *sc = xsc; u_int32_t status; status = vmwpvs_read(sc, VMWPVS_R_INTR_STATUS); if ((status & VMWPVS_INTR_ALL_MASK) == 0) return (0); vmwpvs_write(sc, VMWPVS_R_INTR_STATUS, status); return (vmwpvs_intr(sc)); } int vmwpvs_intr(void *xsc) { struct vmwpvs_softc *sc = xsc; volatile struct vmwpvw_ring_state *s = VMWPVS_DMA_KVA(sc->sc_ring_state); struct vmwpvs_ring_cmp *ring = VMWPVS_DMA_KVA(sc->sc_cmp_ring); struct vmwpvs_ccb_list list = SIMPLEQ_HEAD_INITIALIZER(list); struct vmwpvs_ccb *ccb; u_int32_t cons, prod; int msg; mtx_enter(&sc->sc_ring_mtx); bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0, VMWPVS_PAGE_SIZE, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); cons = s->cmp_cons; prod = s->cmp_prod; s->cmp_cons = prod; msg = (sc->sc_msg_ring != NULL && s->msg_cons != s->msg_prod); bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0, VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); if (cons != prod) { bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_cmp_ring), 0, VMWPVS_PAGE_SIZE, BUS_DMASYNC_POSTREAD); do { ccb = vmwpvs_scsi_cmd_done(sc, &ring[cons++ % VMWPVS_CMP_COUNT]); SIMPLEQ_INSERT_TAIL(&list, ccb, ccb_entry); } while (cons != prod); bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_cmp_ring), 0, VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD); } mtx_leave(&sc->sc_ring_mtx); while ((ccb = SIMPLEQ_FIRST(&list)) != NULL) { SIMPLEQ_REMOVE_HEAD(&list, ccb_entry); scsi_done(ccb->ccb_xs); } if (msg) task_add(systq, &sc->sc_msg_task); return (1); } void vmwpvs_msg_task(void *xsc) { struct vmwpvs_softc *sc = xsc; volatile struct vmwpvw_ring_state *s = VMWPVS_DMA_KVA(sc->sc_ring_state); struct vmwpvs_ring_msg *ring = VMWPVS_DMA_KVA(sc->sc_msg_ring); struct vmwpvs_ring_msg *msg; struct vmwpvs_ring_msg_dev *dvmsg; u_int32_t cons, prod; mtx_enter(&sc->sc_ring_mtx); bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0, VMWPVS_PAGE_SIZE, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); cons = s->msg_cons; prod = s->msg_prod; bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0, VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); mtx_leave(&sc->sc_ring_mtx); /* * we dont have to lock around the msg ring cos the system taskq has * only one thread. */ bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_msg_ring), 0, VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE, BUS_DMASYNC_POSTREAD); while (cons != prod) { msg = &ring[cons++ % VMWPVS_MSG_COUNT]; switch (letoh32(msg->type)) { case VMWPVS_MSG_T_ADDED: dvmsg = (struct vmwpvs_ring_msg_dev *)msg; if (letoh32(dvmsg->bus) != 0) { printf("%s: ignoring request to add device" " on bus %d\n", DEVNAME(sc), letoh32(msg->type)); break; } if (scsi_probe_lun(sc->sc_scsibus, letoh32(dvmsg->target), dvmsg->lun[1]) != 0) { printf("%s: error probing target %d lun %d\n", DEVNAME(sc), letoh32(dvmsg->target), dvmsg->lun[1]); }; break; case VMWPVS_MSG_T_REMOVED: dvmsg = (struct vmwpvs_ring_msg_dev *)msg; if (letoh32(dvmsg->bus) != 0) { printf("%s: ignorint request to remove device" " on bus %d\n", DEVNAME(sc), letoh32(msg->type)); break; } if (scsi_detach_lun(sc->sc_scsibus, letoh32(dvmsg->target), dvmsg->lun[1], DETACH_FORCE) != 0) { printf("%s: error detaching target %d lun %d\n", DEVNAME(sc), letoh32(dvmsg->target), dvmsg->lun[1]); }; break; default: printf("%s: unknown msg type %u\n", DEVNAME(sc), letoh32(msg->type)); break; } } bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_msg_ring), 0, VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD); mtx_enter(&sc->sc_ring_mtx); bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0, VMWPVS_PAGE_SIZE, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); s->msg_cons = prod; bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0, VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); mtx_leave(&sc->sc_ring_mtx); } void vmwpvs_scsi_cmd(struct scsi_xfer *xs) { struct scsi_link *link = xs->sc_link; struct vmwpvs_softc *sc = link->adapter_softc; struct vmwpvs_ccb *ccb = xs->io; bus_dmamap_t dmap = ccb->ccb_dmamap; volatile struct vmwpvw_ring_state *s = VMWPVS_DMA_KVA(sc->sc_ring_state); struct vmwpvs_ring_req *ring = VMWPVS_DMA_KVA(sc->sc_req_ring), *r; u_int32_t prod; struct vmwpvs_ccb_list list; int error; u_int i; ccb->ccb_xs = xs; if (xs->datalen > 0) { error = bus_dmamap_load(sc->sc_dmat, dmap, xs->data, xs->datalen, NULL, (xs->flags & SCSI_NOSLEEP) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK); if (error) { xs->error = XS_DRIVER_STUFFUP; scsi_done(xs); return; } bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, (xs->flags & SCSI_DATA_IN) ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); } mtx_enter(&sc->sc_ring_mtx); bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0, VMWPVS_PAGE_SIZE, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); prod = s->req_prod; r = &ring[prod % VMWPVS_REQ_COUNT]; bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_req_ring), prod * sizeof(*r), sizeof(*r), BUS_DMASYNC_POSTWRITE); memset(r, 0, sizeof(*r)); r->context = ccb->ccb_ctx; if (xs->datalen > 0) { r->data_len = xs->datalen; if (dmap->dm_nsegs == 1) { r->data_addr = dmap->dm_segs[0].ds_addr; } else { struct vmwpvs_sge *sgl = ccb->ccb_sgl->list, *sge; r->data_addr = VMWPVS_DMA_DVA(sc->sc_sgls) + ccb->ccb_sgl_offset; r->flags = VMWPVS_REQ_SGL; for (i = 0; i < dmap->dm_nsegs; i++) { sge = &sgl[i]; sge->addr = dmap->dm_segs[i].ds_addr; sge->len = dmap->dm_segs[i].ds_len; sge->flags = 0; } bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_sgls), ccb->ccb_sgl_offset, sizeof(*sge) * dmap->dm_nsegs, BUS_DMASYNC_PREWRITE); } } r->sense_addr = VMWPVS_DMA_DVA(sc->sc_sense) + ccb->ccb_sense_offset; r->sense_len = sizeof(xs->sense); bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_req_ring), 0, VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE, BUS_DMASYNC_POSTWRITE); switch (xs->flags & (SCSI_DATA_IN | SCSI_DATA_OUT)) { case SCSI_DATA_IN: r->flags |= VMWPVS_REQ_DIR_IN; break; case SCSI_DATA_OUT: r->flags |= VMWPVS_REQ_DIR_OUT; break; default: r->flags |= VMWPVS_REQ_DIR_NONE; break; } memcpy(r->cdb, xs->cmd, xs->cmdlen); r->cdblen = xs->cmdlen; r->lun[1] = link->lun; /* ugly :( */ r->tag = MSG_SIMPLE_Q_TAG; r->bus = 0; r->target = link->target; r->vcpu_hint = 0; bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_req_ring), 0, VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREWRITE); s->req_prod = prod + 1; bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0, VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); vmwpvs_write(sc, xs->bp == NULL ? VMWPVS_R_KICK_NON_RW_IO : VMWPVS_R_KICK_RW_IO, 0); if (!ISSET(xs->flags, SCSI_POLL)) { mtx_leave(&sc->sc_ring_mtx); return; } SIMPLEQ_INIT(&list); do { ccb = vmwpvs_scsi_cmd_poll(sc); SIMPLEQ_INSERT_TAIL(&list, ccb, ccb_entry); } while (xs->io != ccb); mtx_leave(&sc->sc_ring_mtx); while ((ccb = SIMPLEQ_FIRST(&list)) != NULL) { SIMPLEQ_REMOVE_HEAD(&list, ccb_entry); scsi_done(ccb->ccb_xs); } } struct vmwpvs_ccb * vmwpvs_scsi_cmd_poll(struct vmwpvs_softc *sc) { volatile struct vmwpvw_ring_state *s = VMWPVS_DMA_KVA(sc->sc_ring_state); struct vmwpvs_ring_cmp *ring = VMWPVS_DMA_KVA(sc->sc_cmp_ring); struct vmwpvs_ccb *ccb; u_int32_t prod, cons; for (;;) { bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0, VMWPVS_PAGE_SIZE, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); cons = s->cmp_cons; prod = s->cmp_prod; if (cons != prod) s->cmp_cons = cons + 1; bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0, VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); if (cons != prod) break; else delay(1000); } bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_cmp_ring), 0, VMWPVS_PAGE_SIZE * VMWPVS_RING_PAGES, BUS_DMASYNC_POSTREAD); ccb = vmwpvs_scsi_cmd_done(sc, &ring[cons % VMWPVS_CMP_COUNT]); bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_cmp_ring), 0, VMWPVS_PAGE_SIZE * VMWPVS_RING_PAGES, BUS_DMASYNC_PREREAD); return (ccb); } struct vmwpvs_ccb * vmwpvs_scsi_cmd_done(struct vmwpvs_softc *sc, struct vmwpvs_ring_cmp *c) { u_int64_t ctx = c->context; struct vmwpvs_ccb *ccb = &sc->sc_ccbs[ctx & 0xffffffff]; bus_dmamap_t dmap = ccb->ccb_dmamap; struct scsi_xfer *xs = ccb->ccb_xs; bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_sense), ccb->ccb_sense_offset, sizeof(xs->sense), BUS_DMASYNC_POSTREAD); if (xs->datalen > 0) { if (dmap->dm_nsegs > 1) { bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_sgls), ccb->ccb_sgl_offset, sizeof(struct vmwpvs_sge) * dmap->dm_nsegs, BUS_DMASYNC_POSTWRITE); } bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, (xs->flags & SCSI_DATA_IN) ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->sc_dmat, dmap); } xs->status = c->scsi_status; switch (c->host_status) { case VMWPVS_HOST_STATUS_SUCCESS: case VMWPVS_HOST_STATUS_LINKED_CMD_COMPLETED: case VMWPVS_HOST_STATUS_LINKED_CMD_COMPLETED_WITH_FLAG: if (c->scsi_status == VMWPVS_SCSI_STATUS_CHECK) { memcpy(&xs->sense, ccb->ccb_sense, sizeof(xs->sense)); xs->error = XS_SENSE; } else xs->error = XS_NOERROR; xs->resid = 0; break; case VMWPVS_HOST_STATUS_UNDERRUN: case VMWPVS_HOST_STATUS_DATARUN: xs->resid = xs->datalen - c->data_len; xs->error = XS_NOERROR; break; case VMWPVS_HOST_STATUS_SELTIMEOUT: xs->error = XS_SELTIMEOUT; break; default: printf("%s: %s:%d h:0x%x s:0x%x\n", DEVNAME(sc), __FUNCTION__, __LINE__, c->host_status, c->scsi_status); xs->error = XS_DRIVER_STUFFUP; break; } return (ccb); } void * vmwpvs_ccb_get(void *xsc) { struct vmwpvs_softc *sc = xsc; struct vmwpvs_ccb *ccb; mtx_enter(&sc->sc_ccb_mtx); ccb = SIMPLEQ_FIRST(&sc->sc_ccb_list); if (ccb != NULL) SIMPLEQ_REMOVE_HEAD(&sc->sc_ccb_list, ccb_entry); mtx_leave(&sc->sc_ccb_mtx); return (ccb); } void vmwpvs_ccb_put(void *xsc, void *io) { struct vmwpvs_softc *sc = xsc; struct vmwpvs_ccb *ccb = io; mtx_enter(&sc->sc_ccb_mtx); SIMPLEQ_INSERT_HEAD(&sc->sc_ccb_list, ccb, ccb_entry); mtx_leave(&sc->sc_ccb_mtx); } struct vmwpvs_dmamem * vmwpvs_dmamem_alloc(struct vmwpvs_softc *sc, size_t size) { struct vmwpvs_dmamem *dm; int nsegs; dm = malloc(sizeof(*dm), M_DEVBUF, M_NOWAIT | M_ZERO); if (dm == NULL) return (NULL); dm->dm_size = size; if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &dm->dm_map) != 0) goto dmfree; if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &dm->dm_seg, 1, &nsegs, BUS_DMA_NOWAIT | BUS_DMA_ZERO) != 0) goto destroy; if (bus_dmamem_map(sc->sc_dmat, &dm->dm_seg, nsegs, size, &dm->dm_kva, BUS_DMA_NOWAIT) != 0) goto free; if (bus_dmamap_load(sc->sc_dmat, dm->dm_map, dm->dm_kva, size, NULL, BUS_DMA_NOWAIT) != 0) goto unmap; return (dm); unmap: bus_dmamem_unmap(sc->sc_dmat, dm->dm_kva, size); free: bus_dmamem_free(sc->sc_dmat, &dm->dm_seg, 1); destroy: bus_dmamap_destroy(sc->sc_dmat, dm->dm_map); dmfree: free(dm, M_DEVBUF, sizeof *dm); return (NULL); } struct vmwpvs_dmamem * vmwpvs_dmamem_zalloc(struct vmwpvs_softc *sc, size_t size) { struct vmwpvs_dmamem *dm; dm = vmwpvs_dmamem_alloc(sc, size); if (dm == NULL) return (NULL); memset(VMWPVS_DMA_KVA(dm), 0, size); return (dm); } void vmwpvs_dmamem_free(struct vmwpvs_softc *sc, struct vmwpvs_dmamem *dm) { bus_dmamap_unload(sc->sc_dmat, dm->dm_map); bus_dmamem_unmap(sc->sc_dmat, dm->dm_kva, dm->dm_size); bus_dmamem_free(sc->sc_dmat, &dm->dm_seg, 1); bus_dmamap_destroy(sc->sc_dmat, dm->dm_map); free(dm, M_DEVBUF, sizeof *dm); }