/* $OpenBSD: dwpcie.c,v 1.57 2024/09/01 03:08:56 jsg Exp $ */ /* * Copyright (c) 2018 Mark Kettenis * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* Registers */ #define PCIE_PORT_LINK_CTRL 0x710 #define PCIE_PORT_LINK_CTRL_LANES_MASK (0x3f << 16) #define PCIE_PORT_LINK_CTRL_LANES_1 (0x1 << 16) #define PCIE_PORT_LINK_CTRL_LANES_2 (0x3 << 16) #define PCIE_PORT_LINK_CTRL_LANES_4 (0x7 << 16) #define PCIE_PORT_LINK_CTRL_LANES_8 (0xf << 16) #define PCIE_PHY_DEBUG_R1 0x72c #define PCIE_PHY_DEBUG_R1_XMLH_LINK_IN_TRAINING (1 << 29) #define PCIE_PHY_DEBUG_R1_XMLH_LINK_UP (1 << 4) #define PCIE_LINK_WIDTH_SPEED_CTRL 0x80c #define PCIE_LINK_WIDTH_SPEED_CTRL_LANES_MASK (0x1f << 8) #define PCIE_LINK_WIDTH_SPEED_CTRL_LANES_1 (0x1 << 8) #define PCIE_LINK_WIDTH_SPEED_CTRL_LANES_2 (0x2 << 8) #define PCIE_LINK_WIDTH_SPEED_CTRL_LANES_4 (0x4 << 8) #define PCIE_LINK_WIDTH_SPEED_CTRL_LANES_8 (0x8 << 8) #define PCIE_LINK_WIDTH_SPEED_CTRL_CHANGE (1 << 17) #define PCIE_MSI_ADDR_LO 0x820 #define PCIE_MSI_ADDR_HI 0x824 #define PCIE_MSI_INTR_ENABLE(x) (0x828 + (x) * 12) #define PCIE_MSI_INTR_MASK(x) (0x82c + (x) * 12) #define PCIE_MSI_INTR_STATUS(x) (0x830 + (x) * 12) #define MISC_CONTROL_1 0x8bc #define MISC_CONTROL_1_DBI_RO_WR_EN (1 << 0) #define IATU_VIEWPORT 0x900 #define IATU_VIEWPORT_INDEX0 0 #define IATU_VIEWPORT_INDEX1 1 #define IATU_VIEWPORT_INDEX2 2 #define IATU_VIEWPORT_INDEX3 3 #define IATU_OFFSET_VIEWPORT 0x904 #define IATU_OFFSET_UNROLL(x) (0x200 * (x)) #define IATU_REGION_CTRL_1 0x000 #define IATU_REGION_CTRL_1_TYPE_MEM 0 #define IATU_REGION_CTRL_1_TYPE_IO 2 #define IATU_REGION_CTRL_1_TYPE_CFG0 4 #define IATU_REGION_CTRL_1_TYPE_CFG1 5 #define IATU_REGION_CTRL_2 0x004 #define IATU_REGION_CTRL_2_REGION_EN (1U << 31) #define IATU_LWR_BASE_ADDR 0x08 #define IATU_UPPER_BASE_ADDR 0x0c #define IATU_LIMIT_ADDR 0x10 #define IATU_LWR_TARGET_ADDR 0x14 #define IATU_UPPER_TARGET_ADDR 0x18 /* Marvell ARMADA 8k registers */ #define PCIE_GLOBAL_CTRL 0x8000 #define PCIE_GLOBAL_CTRL_APP_LTSSM_EN (1 << 2) #define PCIE_GLOBAL_CTRL_DEVICE_TYPE_MASK (0xf << 4) #define PCIE_GLOBAL_CTRL_DEVICE_TYPE_RC (0x4 << 4) #define PCIE_GLOBAL_STATUS 0x8008 #define PCIE_GLOBAL_STATUS_RDLH_LINK_UP (1 << 1) #define PCIE_GLOBAL_STATUS_PHY_LINK_UP (1 << 9) #define PCIE_PM_STATUS 0x8014 #define PCIE_GLOBAL_INT_CAUSE 0x801c #define PCIE_GLOBAL_INT_MASK 0x8020 #define PCIE_GLOBAL_INT_MASK_INT_A (1 << 9) #define PCIE_GLOBAL_INT_MASK_INT_B (1 << 10) #define PCIE_GLOBAL_INT_MASK_INT_C (1 << 11) #define PCIE_GLOBAL_INT_MASK_INT_D (1 << 12) #define PCIE_ARCACHE_TRC 0x8050 #define PCIE_ARCACHE_TRC_DEFAULT 0x3511 #define PCIE_AWCACHE_TRC 0x8054 #define PCIE_AWCACHE_TRC_DEFAULT 0x5311 #define PCIE_ARUSER 0x805c #define PCIE_AWUSER 0x8060 #define PCIE_AXUSER_DOMAIN_MASK (0x3 << 4) #define PCIE_AXUSER_DOMAIN_INNER_SHARABLE (0x1 << 4) #define PCIE_AXUSER_DOMAIN_OUTER_SHARABLE (0x2 << 4) #define PCIE_STREAMID 0x8064 #define PCIE_STREAMID_FUNC_BITS(x) ((x) << 0) #define PCIE_STREAMID_DEV_BITS(x) ((x) << 4) #define PCIE_STREAMID_BUS_BITS(x) ((x) << 8) #define PCIE_STREAMID_ROOTPORT(x) ((x) << 12) #define PCIE_STREAMID_8040 \ (PCIE_STREAMID_ROOTPORT(0x80) | PCIE_STREAMID_BUS_BITS(2) | \ PCIE_STREAMID_DEV_BITS(2) | PCIE_STREAMID_FUNC_BITS(3)) /* Amlogic G12A registers */ #define PCIE_CFG0 0x0000 #define PCIE_CFG0_APP_LTSSM_EN (1 << 7) #define PCIE_STATUS12 0x0030 #define PCIE_STATUS12_RDLH_LINK_UP (1 << 16) #define PCIE_STATUS12_LTSSM_MASK (0x1f << 10) #define PCIE_STATUS12_LTSSM_UP (0x11 << 10) #define PCIE_STATUS12_SMLH_LINK_UP (1 << 6) /* NXP i.MX8MQ registers */ #define PCIE_RC_LCR 0x7c #define PCIE_RC_LCR_MAX_LINK_SPEEDS_GEN1 0x1 #define PCIE_RC_LCR_MAX_LINK_SPEEDS_GEN2 0x2 #define PCIE_RC_LCR_MAX_LINK_SPEEDS_MASK 0xf #define PCIE_RC_LCR_L1EL_MASK (0x7 << 15) #define PCIE_RC_LCR_L1EL_64US (0x6 << 15) #define IOMUXC_GPR12 0x30 #define IMX8MQ_GPR_PCIE2_DEVICE_TYPE_MASK (0xf << 8) #define IMX8MQ_GPR_PCIE2_DEVICE_TYPE_RC (0x4 << 8) #define IMX8MQ_GPR_PCIE1_DEVICE_TYPE_MASK (0xf << 12) #define IMX8MQ_GPR_PCIE1_DEVICE_TYPE_RC (0x4 << 12) #define IOMUXC_GPR14 0x38 #define IOMUXC_GPR16 0x40 #define IMX8MQ_GPR_PCIE_REF_USE_PAD (1 << 9) #define IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE_EN (1 << 10) #define IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE (1 << 11) #define IMX8MM_GPR_PCIE_SSC_EN (1 << 16) #define IMX8MM_GPR_PCIE_POWER_OFF (1 << 17) #define IMX8MM_GPR_PCIE_CMN_RST (1 << 18) #define IMX8MM_GPR_PCIE_AUX_EN (1 << 19) #define IMX8MM_GPR_PCIE_REF_CLK_MASK (0x3 << 24) #define IMX8MM_GPR_PCIE_REF_CLK_PLL (0x3 << 24) #define IMX8MM_GPR_PCIE_REF_CLK_EXT (0x2 << 24) #define IMX8MM_PCIE_PHY_CMN_REG62 0x188 #define IMX8MM_PCIE_PHY_CMN_REG62_PLL_CLK_OUT 0x08 #define IMX8MM_PCIE_PHY_CMN_REG64 0x190 #define IMX8MM_PCIE_PHY_CMN_REG64_AUX_RX_TX_TERM 0x8c #define IMX8MM_PCIE_PHY_CMN_REG75 0x1d4 #define IMX8MM_PCIE_PHY_CMN_REG75_PLL_DONE 0x3 #define IMX8MM_PCIE_PHY_TRSV_REG5 0x414 #define IMX8MM_PCIE_PHY_TRSV_REG5_GEN1_DEEMP 0x2d #define IMX8MM_PCIE_PHY_TRSV_REG6 0x418 #define IMX8MM_PCIE_PHY_TRSV_REG6_GEN2_DEEMP 0xf #define ANATOP_PLLOUT_CTL 0x74 #define ANATOP_PLLOUT_CTL_CKE (1 << 4) #define ANATOP_PLLOUT_CTL_SEL_SYSPLL1 0xb #define ANATOP_PLLOUT_CTL_SEL_MASK 0xf #define ANATOP_PLLOUT_DIV 0x7c #define ANATOP_PLLOUT_DIV_SYSPLL1 0x7 /* Rockchip RK3568/RK3588 registers */ #define PCIE_CLIENT_GENERAL_CON 0x0000 #define PCIE_CLIENT_DEV_TYPE_RC ((0xf << 4) << 16 | (0x4 << 4)) #define PCIE_CLIENT_LINK_REQ_RST_GRT ((1 << 3) << 16 | (1 << 3)) #define PCIE_CLIENT_APP_LTSSM_ENABLE ((1 << 2) << 16 | (1 << 2)) #define PCIE_CLIENT_INTR_STATUS_LEGACY 0x0008 #define PCIE_CLIENT_INTR_MASK_LEGACY 0x001c #define PCIE_CLIENT_HOT_RESET_CTRL 0x0180 #define PCIE_CLIENT_APP_LTSSM_ENABLE_ENHANCE ((1 << 4) << 16 | (1 << 4)) #define PCIE_CLIENT_LTSSM_STATUS 0x0300 #define PCIE_CLIENT_RDLH_LINK_UP (1 << 17) #define PCIE_CLIENT_SMLH_LINK_UP (1 << 16) #define PCIE_CLIENT_LTSSM_MASK (0x1f << 0) #define PCIE_CLIENT_LTSSM_UP (0x11 << 0) #define HREAD4(sc, reg) \ (bus_space_read_4((sc)->sc_iot, (sc)->sc_ioh, (reg))) #define HWRITE4(sc, reg, val) \ bus_space_write_4((sc)->sc_iot, (sc)->sc_ioh, (reg), (val)) #define HSET4(sc, reg, bits) \ HWRITE4((sc), (reg), HREAD4((sc), (reg)) | (bits)) #define HCLR4(sc, reg, bits) \ HWRITE4((sc), (reg), HREAD4((sc), (reg)) & ~(bits)) struct dwpcie_range { uint32_t flags; uint64_t pci_base; uint64_t phys_base; uint64_t size; }; struct dwpcie_intx { int (*di_func)(void *); void *di_arg; int di_ipl; int di_flags; int di_pin; struct evcount di_count; char *di_name; struct dwpcie_softc *di_sc; TAILQ_ENTRY(dwpcie_intx) di_next; }; #define DWPCIE_MAX_MSI 64 struct dwpcie_msi { int (*dm_func)(void *); void *dm_arg; int dm_ipl; int dm_flags; int dm_vec; int dm_nvec; struct evcount dm_count; char *dm_name; }; struct dwpcie_softc { struct device sc_dev; bus_space_tag_t sc_iot; bus_space_handle_t sc_ioh; bus_dma_tag_t sc_dmat; bus_addr_t sc_ctrl_base; bus_size_t sc_ctrl_size; bus_addr_t sc_conf_base; bus_size_t sc_conf_size; bus_space_handle_t sc_conf_ioh; bus_addr_t sc_glue_base; bus_size_t sc_glue_size; bus_space_handle_t sc_glue_ioh; bus_addr_t sc_atu_base; bus_size_t sc_atu_size; bus_space_handle_t sc_atu_ioh; bus_addr_t sc_io_base; bus_addr_t sc_io_bus_addr; bus_size_t sc_io_size; bus_addr_t sc_mem_base; bus_addr_t sc_mem_bus_addr; bus_size_t sc_mem_size; bus_addr_t sc_pmem_base; bus_addr_t sc_pmem_bus_addr; bus_size_t sc_pmem_size; int sc_node; int sc_acells; int sc_scells; int sc_pacells; int sc_pscells; struct dwpcie_range *sc_ranges; int sc_nranges; struct bus_space sc_bus_iot; struct bus_space sc_bus_memt; struct machine_pci_chipset sc_pc; int sc_bus; int sc_num_viewport; int sc_atu_unroll; int sc_atu_viewport; void *sc_ih; struct interrupt_controller sc_ic; TAILQ_HEAD(,dwpcie_intx) sc_intx[4]; void *sc_msi_ih[2]; uint64_t sc_msi_addr; uint64_t sc_msi_mask; struct dwpcie_msi sc_msi[DWPCIE_MAX_MSI]; int sc_num_msi; }; struct dwpcie_intr_handle { struct machine_intr_handle pih_ih; struct dwpcie_softc *pih_sc; struct dwpcie_msi *pih_dm; bus_dma_tag_t pih_dmat; bus_dmamap_t pih_map; }; int dwpcie_match(struct device *, void *, void *); void dwpcie_attach(struct device *, struct device *, void *); const struct cfattach dwpcie_ca = { sizeof (struct dwpcie_softc), dwpcie_match, dwpcie_attach }; struct cfdriver dwpcie_cd = { NULL, "dwpcie", DV_DULL }; int dwpcie_match(struct device *parent, void *match, void *aux) { struct fdt_attach_args *faa = aux; return (OF_is_compatible(faa->fa_node, "amlogic,g12a-pcie") || OF_is_compatible(faa->fa_node, "baikal,bm1000-pcie") || OF_is_compatible(faa->fa_node, "fsl,imx8mm-pcie") || OF_is_compatible(faa->fa_node, "fsl,imx8mq-pcie") || OF_is_compatible(faa->fa_node, "marvell,armada8k-pcie") || OF_is_compatible(faa->fa_node, "qcom,pcie-sc8280xp") || OF_is_compatible(faa->fa_node, "qcom,pcie-x1e80100") || OF_is_compatible(faa->fa_node, "rockchip,rk3568-pcie") || OF_is_compatible(faa->fa_node, "rockchip,rk3588-pcie") || OF_is_compatible(faa->fa_node, "sifive,fu740-pcie")); } void dwpcie_attach_deferred(struct device *); void dwpcie_atu_disable(struct dwpcie_softc *, int); void dwpcie_atu_config(struct dwpcie_softc *, int, int, uint64_t, uint64_t, uint64_t); void dwpcie_link_config(struct dwpcie_softc *); int dwpcie_link_up(struct dwpcie_softc *); int dwpcie_armada8k_init(struct dwpcie_softc *); int dwpcie_armada8k_link_up(struct dwpcie_softc *); int dwpcie_armada8k_intr(void *); int dwpcie_g12a_init(struct dwpcie_softc *); int dwpcie_g12a_link_up(struct dwpcie_softc *); int dwpcie_imx8mq_init(struct dwpcie_softc *); int dwpcie_imx8mq_intr(void *); int dwpcie_fu740_init(struct dwpcie_softc *); int dwpcie_rk3568_init(struct dwpcie_softc *); int dwpcie_rk3568_intr(void *); void *dwpcie_rk3568_intr_establish(void *, int *, int, struct cpu_info *, int (*)(void *), void *, char *); void dwpcie_rk3568_intr_disestablish(void *); void dwpcie_rk3568_intr_barrier(void *); int dwpcie_sc8280xp_init(struct dwpcie_softc *); void dwpcie_attach_hook(struct device *, struct device *, struct pcibus_attach_args *); int dwpcie_bus_maxdevs(void *, int); pcitag_t dwpcie_make_tag(void *, int, int, int); void dwpcie_decompose_tag(void *, pcitag_t, int *, int *, int *); int dwpcie_conf_size(void *, pcitag_t); pcireg_t dwpcie_conf_read(void *, pcitag_t, int); void dwpcie_conf_write(void *, pcitag_t, int, pcireg_t); int dwpcie_probe_device_hook(void *, struct pci_attach_args *); int dwpcie_intr_map(struct pci_attach_args *, pci_intr_handle_t *); const char *dwpcie_intr_string(void *, pci_intr_handle_t); void *dwpcie_intr_establish(void *, pci_intr_handle_t, int, struct cpu_info *, int (*)(void *), void *, char *); void dwpcie_intr_disestablish(void *, void *); int dwpcie_bs_iomap(bus_space_tag_t, bus_addr_t, bus_size_t, int, bus_space_handle_t *); int dwpcie_bs_memmap(bus_space_tag_t, bus_addr_t, bus_size_t, int, bus_space_handle_t *); struct interrupt_controller dwpcie_ic = { .ic_barrier = intr_barrier }; void dwpcie_attach(struct device *parent, struct device *self, void *aux) { struct dwpcie_softc *sc = (struct dwpcie_softc *)self; struct fdt_attach_args *faa = aux; uint32_t *ranges; int i, j, nranges, rangeslen; int atu, config, ctrl, glue; if (faa->fa_nreg < 2) { printf(": no registers\n"); return; } sc->sc_ctrl_base = faa->fa_reg[0].addr; sc->sc_ctrl_size = faa->fa_reg[0].size; ctrl = OF_getindex(faa->fa_node, "dbi", "reg-names"); if (ctrl >= 0 && ctrl < faa->fa_nreg) { sc->sc_ctrl_base = faa->fa_reg[ctrl].addr; sc->sc_ctrl_size = faa->fa_reg[ctrl].size; } config = OF_getindex(faa->fa_node, "config", "reg-names"); if (config < 0 || config >= faa->fa_nreg) { printf(": no config registers\n"); return; } sc->sc_conf_base = faa->fa_reg[config].addr; sc->sc_conf_size = faa->fa_reg[config].size; sc->sc_atu_base = sc->sc_ctrl_base + 0x300000; sc->sc_atu_size = sc->sc_ctrl_size - 0x300000; atu = OF_getindex(faa->fa_node, "atu", "reg-names"); if (atu >= 0 && atu < faa->fa_nreg) { sc->sc_atu_base = faa->fa_reg[atu].addr; sc->sc_atu_size = faa->fa_reg[atu].size; } if (OF_is_compatible(faa->fa_node, "amlogic,g12a-pcie")) { glue = OF_getindex(faa->fa_node, "cfg", "reg-names"); if (glue < 0 || glue >= faa->fa_nreg) { printf(": no glue registers\n"); return; } sc->sc_glue_base = faa->fa_reg[glue].addr; sc->sc_glue_size = faa->fa_reg[glue].size; } if (OF_is_compatible(faa->fa_node, "rockchip,rk3568-pcie") || OF_is_compatible(faa->fa_node, "rockchip,rk3588-pcie")) { glue = OF_getindex(faa->fa_node, "apb", "reg-names"); if (glue < 0 || glue >= faa->fa_nreg) { printf(": no glue registers\n"); return; } sc->sc_glue_base = faa->fa_reg[glue].addr; sc->sc_glue_size = faa->fa_reg[glue].size; } sc->sc_iot = faa->fa_iot; sc->sc_dmat = faa->fa_dmat; sc->sc_node = faa->fa_node; sc->sc_acells = OF_getpropint(sc->sc_node, "#address-cells", faa->fa_acells); sc->sc_scells = OF_getpropint(sc->sc_node, "#size-cells", faa->fa_scells); sc->sc_pacells = faa->fa_acells; sc->sc_pscells = faa->fa_scells; rangeslen = OF_getproplen(sc->sc_node, "ranges"); if (rangeslen <= 0 || (rangeslen % sizeof(uint32_t)) || (rangeslen / sizeof(uint32_t)) % (sc->sc_acells + sc->sc_pacells + sc->sc_scells)) { printf(": invalid ranges property\n"); return; } ranges = malloc(rangeslen, M_TEMP, M_WAITOK); OF_getpropintarray(sc->sc_node, "ranges", ranges, rangeslen); nranges = (rangeslen / sizeof(uint32_t)) / (sc->sc_acells + sc->sc_pacells + sc->sc_scells); sc->sc_ranges = mallocarray(nranges, sizeof(struct dwpcie_range), M_TEMP, M_WAITOK); sc->sc_nranges = nranges; for (i = 0, j = 0; i < sc->sc_nranges; i++) { sc->sc_ranges[i].flags = ranges[j++]; sc->sc_ranges[i].pci_base = ranges[j++]; if (sc->sc_acells - 1 == 2) { sc->sc_ranges[i].pci_base <<= 32; sc->sc_ranges[i].pci_base |= ranges[j++]; } sc->sc_ranges[i].phys_base = ranges[j++]; if (sc->sc_pacells == 2) { sc->sc_ranges[i].phys_base <<= 32; sc->sc_ranges[i].phys_base |= ranges[j++]; } sc->sc_ranges[i].size = ranges[j++]; if (sc->sc_scells == 2) { sc->sc_ranges[i].size <<= 32; sc->sc_ranges[i].size |= ranges[j++]; } } free(ranges, M_TEMP, rangeslen); if (bus_space_map(sc->sc_iot, sc->sc_ctrl_base, sc->sc_ctrl_size, 0, &sc->sc_ioh)) { free(sc->sc_ranges, M_TEMP, sc->sc_nranges * sizeof(struct dwpcie_range)); printf(": can't map ctrl registers\n"); return; } if (bus_space_map(sc->sc_iot, sc->sc_conf_base, sc->sc_conf_size, 0, &sc->sc_conf_ioh)) { bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_ctrl_size); free(sc->sc_ranges, M_TEMP, sc->sc_nranges * sizeof(struct dwpcie_range)); printf(": can't map config registers\n"); return; } sc->sc_num_viewport = OF_getpropint(sc->sc_node, "num-viewport", 2); printf("\n"); pinctrl_byname(sc->sc_node, "default"); clock_set_assigned(sc->sc_node); config_defer(self, dwpcie_attach_deferred); } void dwpcie_attach_deferred(struct device *self) { struct dwpcie_softc *sc = (struct dwpcie_softc *)self; struct pcibus_attach_args pba; bus_addr_t iobase, iolimit; bus_addr_t membase, memlimit; bus_addr_t pmembase, pmemlimit; uint32_t bus_range[2]; pcireg_t bir, blr, csr; int i, error = 0; if (OF_is_compatible(sc->sc_node, "marvell,armada8k-pcie")) error = dwpcie_armada8k_init(sc); if (OF_is_compatible(sc->sc_node, "amlogic,g12a-pcie")) error = dwpcie_g12a_init(sc); if (OF_is_compatible(sc->sc_node, "fsl,imx8mm-pcie") || OF_is_compatible(sc->sc_node, "fsl,imx8mq-pcie")) error = dwpcie_imx8mq_init(sc); if (OF_is_compatible(sc->sc_node, "qcom,pcie-sc8280xp") || OF_is_compatible(sc->sc_node, "qcom,pcie-x1e80100")) error = dwpcie_sc8280xp_init(sc); if (OF_is_compatible(sc->sc_node, "rockchip,rk3568-pcie") || OF_is_compatible(sc->sc_node, "rockchip,rk3588-pcie")) error = dwpcie_rk3568_init(sc); if (OF_is_compatible(sc->sc_node, "sifive,fu740-pcie")) error = dwpcie_fu740_init(sc); if (error != 0) { bus_space_unmap(sc->sc_iot, sc->sc_conf_ioh, sc->sc_conf_size); bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_ctrl_size); free(sc->sc_ranges, M_TEMP, sc->sc_nranges * sizeof(struct dwpcie_range)); printf("%s: can't initialize hardware\n", sc->sc_dev.dv_xname); return; } sc->sc_atu_viewport = -1; if (HREAD4(sc, IATU_VIEWPORT) == 0xffffffff) { sc->sc_atu_unroll = 1; if (bus_space_map(sc->sc_iot, sc->sc_atu_base, sc->sc_atu_size, 0, &sc->sc_atu_ioh)) { bus_space_unmap(sc->sc_iot, sc->sc_conf_ioh, sc->sc_conf_size); bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_ctrl_size); free(sc->sc_ranges, M_TEMP, sc->sc_nranges * sizeof(struct dwpcie_range)); printf("%s: can't map atu registers\n", sc->sc_dev.dv_xname); return; } } /* Set up address translation for I/O space. */ for (i = 0; i < sc->sc_nranges; i++) { if ((sc->sc_ranges[i].flags & 0x03000000) == 0x01000000 && sc->sc_ranges[i].size > 0) { sc->sc_io_base = sc->sc_ranges[i].phys_base; sc->sc_io_bus_addr = sc->sc_ranges[i].pci_base; sc->sc_io_size = sc->sc_ranges[i].size; } if ((sc->sc_ranges[i].flags & 0x03000000) == 0x02000000 && sc->sc_ranges[i].size > 0) { sc->sc_mem_base = sc->sc_ranges[i].phys_base; sc->sc_mem_bus_addr = sc->sc_ranges[i].pci_base; sc->sc_mem_size = sc->sc_ranges[i].size; } if ((sc->sc_ranges[i].flags & 0x03000000) == 0x03000000 && sc->sc_ranges[i].size > 0) { sc->sc_pmem_base = sc->sc_ranges[i].phys_base; sc->sc_pmem_bus_addr = sc->sc_ranges[i].pci_base; sc->sc_pmem_size = sc->sc_ranges[i].size; } } if (sc->sc_mem_size == 0) { printf("%s: no memory mapped I/O window\n", sc->sc_dev.dv_xname); return; } /* * Disable prefetchable memory mapped I/O window if we don't * have enough viewports to enable it. */ if (sc->sc_num_viewport < 4) sc->sc_pmem_size = 0; for (i = 0; i < sc->sc_num_viewport; i++) dwpcie_atu_disable(sc, i); dwpcie_atu_config(sc, IATU_VIEWPORT_INDEX0, IATU_REGION_CTRL_1_TYPE_MEM, sc->sc_mem_base, sc->sc_mem_bus_addr, sc->sc_mem_size); if (sc->sc_num_viewport > 2 && sc->sc_io_size > 0) dwpcie_atu_config(sc, IATU_VIEWPORT_INDEX2, IATU_REGION_CTRL_1_TYPE_IO, sc->sc_io_base, sc->sc_io_bus_addr, sc->sc_io_size); if (sc->sc_num_viewport > 3 && sc->sc_pmem_size > 0) dwpcie_atu_config(sc, IATU_VIEWPORT_INDEX3, IATU_REGION_CTRL_1_TYPE_MEM, sc->sc_pmem_base, sc->sc_pmem_bus_addr, sc->sc_pmem_size); /* Enable modification of read-only bits. */ HSET4(sc, MISC_CONTROL_1, MISC_CONTROL_1_DBI_RO_WR_EN); /* A Root Port is a PCI-PCI Bridge. */ HWRITE4(sc, PCI_CLASS_REG, PCI_CLASS_BRIDGE << PCI_CLASS_SHIFT | PCI_SUBCLASS_BRIDGE_PCI << PCI_SUBCLASS_SHIFT); /* Clear BAR as U-Boot seems to leave garbage in it. */ HWRITE4(sc, PCI_MAPREG_START, PCI_MAPREG_MEM_TYPE_64BIT); HWRITE4(sc, PCI_MAPREG_START + 4, 0); /* Enable 32-bit I/O addressing. */ HSET4(sc, PPB_REG_IOSTATUS, PPB_IO_32BIT | (PPB_IO_32BIT << PPB_IOLIMIT_SHIFT)); /* Make sure read-only bits are write-protected. */ HCLR4(sc, MISC_CONTROL_1, MISC_CONTROL_1_DBI_RO_WR_EN); /* Set up bus range. */ if (OF_getpropintarray(sc->sc_node, "bus-range", bus_range, sizeof(bus_range)) != sizeof(bus_range)) { bus_range[0] = 0; bus_range[1] = 31; } sc->sc_bus = bus_range[0]; /* Initialize bus range. */ bir = bus_range[0]; bir |= ((bus_range[0] + 1) << 8); bir |= (bus_range[1] << 16); HWRITE4(sc, PPB_REG_BUSINFO, bir); /* Initialize memory mapped I/O window. */ membase = sc->sc_mem_bus_addr; memlimit = membase + sc->sc_mem_size - 1; blr = memlimit & PPB_MEM_MASK; blr |= (membase >> PPB_MEM_SHIFT); HWRITE4(sc, PPB_REG_MEM, blr); /* Initialize I/O window. */ if (sc->sc_io_size > 0) { iobase = sc->sc_io_bus_addr; iolimit = iobase + sc->sc_io_size - 1; blr = iolimit & PPB_IO_MASK; blr |= (iobase >> PPB_IO_SHIFT); HWRITE4(sc, PPB_REG_IOSTATUS, blr); blr = (iobase & 0xffff0000) >> 16; blr |= iolimit & 0xffff0000; HWRITE4(sc, PPB_REG_IO_HI, blr); } else { HWRITE4(sc, PPB_REG_IOSTATUS, 0x000000ff); HWRITE4(sc, PPB_REG_IO_HI, 0x0000ffff); } /* Initialize prefetchable memory mapped I/O window. */ if (sc->sc_pmem_size > 0) { pmembase = sc->sc_pmem_bus_addr; pmemlimit = pmembase + sc->sc_pmem_size - 1; blr = pmemlimit & PPB_MEM_MASK; blr |= ((pmembase & PPB_MEM_MASK) >> PPB_MEM_SHIFT); HWRITE4(sc, PPB_REG_PREFMEM, blr); HWRITE4(sc, PPB_REG_PREFBASE_HI32, pmembase >> 32); HWRITE4(sc, PPB_REG_PREFLIM_HI32, pmemlimit >> 32); } else { HWRITE4(sc, PPB_REG_PREFMEM, 0x0000ffff); HWRITE4(sc, PPB_REG_PREFBASE_HI32, 0); HWRITE4(sc, PPB_REG_PREFLIM_HI32, 0); } csr = PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_MEM_ENABLE; if (sc->sc_io_size > 0) csr |= PCI_COMMAND_IO_ENABLE; HWRITE4(sc, PCI_COMMAND_STATUS_REG, csr); memcpy(&sc->sc_bus_iot, sc->sc_iot, sizeof(sc->sc_bus_iot)); sc->sc_bus_iot.bus_private = sc; sc->sc_bus_iot._space_map = dwpcie_bs_iomap; memcpy(&sc->sc_bus_memt, sc->sc_iot, sizeof(sc->sc_bus_memt)); sc->sc_bus_memt.bus_private = sc; sc->sc_bus_memt._space_map = dwpcie_bs_memmap; sc->sc_pc.pc_conf_v = sc; sc->sc_pc.pc_attach_hook = dwpcie_attach_hook; sc->sc_pc.pc_bus_maxdevs = dwpcie_bus_maxdevs; sc->sc_pc.pc_make_tag = dwpcie_make_tag; sc->sc_pc.pc_decompose_tag = dwpcie_decompose_tag; sc->sc_pc.pc_conf_size = dwpcie_conf_size; sc->sc_pc.pc_conf_read = dwpcie_conf_read; sc->sc_pc.pc_conf_write = dwpcie_conf_write; sc->sc_pc.pc_probe_device_hook = dwpcie_probe_device_hook; sc->sc_pc.pc_intr_v = sc; sc->sc_pc.pc_intr_map = dwpcie_intr_map; sc->sc_pc.pc_intr_map_msi = _pci_intr_map_msi; sc->sc_pc.pc_intr_map_msivec = _pci_intr_map_msivec; sc->sc_pc.pc_intr_map_msix = _pci_intr_map_msix; sc->sc_pc.pc_intr_string = dwpcie_intr_string; sc->sc_pc.pc_intr_establish = dwpcie_intr_establish; sc->sc_pc.pc_intr_disestablish = dwpcie_intr_disestablish; memset(&pba, 0, sizeof(pba)); pba.pba_busname = "pci"; pba.pba_iot = &sc->sc_bus_iot; pba.pba_memt = &sc->sc_bus_memt; pba.pba_dmat = sc->sc_dmat; pba.pba_pc = &sc->sc_pc; pba.pba_domain = pci_ndomains++; pba.pba_bus = sc->sc_bus; if (OF_is_compatible(sc->sc_node, "baikal,bm1000-pcie") || OF_is_compatible(sc->sc_node, "marvell,armada8k-pcie") || OF_getproplen(sc->sc_node, "msi-map") > 0 || sc->sc_msi_addr) pba.pba_flags |= PCI_FLAGS_MSI_ENABLED; /* * Only support multiple MSI vectors if we have enough MSI * interrupts (or are using an external interrupt controller * that hopefully supports plenty of MSI interrupts). */ if (OF_getproplen(sc->sc_node, "msi-map") > 0 || sc->sc_num_msi > 32) pba.pba_flags |= PCI_FLAGS_MSIVEC_ENABLED; pci_dopm = 1; config_found(self, &pba, NULL); } void dwpcie_link_config(struct dwpcie_softc *sc) { uint32_t mode, width, reg; int lanes; lanes = OF_getpropint(sc->sc_node, "num-lanes", 0); switch (lanes) { case 1: mode = PCIE_PORT_LINK_CTRL_LANES_1; width = PCIE_LINK_WIDTH_SPEED_CTRL_LANES_1; break; case 2: mode = PCIE_PORT_LINK_CTRL_LANES_2; width = PCIE_LINK_WIDTH_SPEED_CTRL_LANES_2; break; case 4: mode = PCIE_PORT_LINK_CTRL_LANES_4; width = PCIE_LINK_WIDTH_SPEED_CTRL_LANES_4; break; case 8: mode = PCIE_PORT_LINK_CTRL_LANES_8; width = PCIE_LINK_WIDTH_SPEED_CTRL_LANES_8; break; default: printf("%s: %d lanes not supported\n", __func__, lanes); return; } reg = HREAD4(sc, PCIE_PORT_LINK_CTRL); reg &= ~PCIE_PORT_LINK_CTRL_LANES_MASK; reg |= mode; HWRITE4(sc, PCIE_PORT_LINK_CTRL, reg); reg = HREAD4(sc, PCIE_LINK_WIDTH_SPEED_CTRL); reg &= ~PCIE_LINK_WIDTH_SPEED_CTRL_LANES_MASK; reg |= width; HWRITE4(sc, PCIE_LINK_WIDTH_SPEED_CTRL, reg); reg = HREAD4(sc, PCIE_LINK_WIDTH_SPEED_CTRL); reg |= PCIE_LINK_WIDTH_SPEED_CTRL_CHANGE; HWRITE4(sc, PCIE_LINK_WIDTH_SPEED_CTRL, reg); } int dwpcie_msi_intr(struct dwpcie_softc *sc, int idx) { struct dwpcie_msi *dm; uint32_t status; int vec, s; status = HREAD4(sc, PCIE_MSI_INTR_STATUS(idx)); if (status == 0) return 0; HWRITE4(sc, PCIE_MSI_INTR_STATUS(idx), status); while (status) { vec = ffs(status) - 1; status &= ~(1U << vec); dm = &sc->sc_msi[idx * 32 + vec]; if (dm->dm_func == NULL) continue; if ((dm->dm_flags & IPL_MPSAFE) == 0) KERNEL_LOCK(); s = splraise(dm->dm_ipl); if (dm->dm_func(dm->dm_arg)) dm->dm_count.ec_count++; splx(s); if ((dm->dm_flags & IPL_MPSAFE) == 0) KERNEL_UNLOCK(); } return 1; } int dwpcie_msi0_intr(void *arg) { return dwpcie_msi_intr(arg, 0); } int dwpcie_msi1_intr(void *arg) { return dwpcie_msi_intr(arg, 1); } int dwpcie_msi_init(struct dwpcie_softc *sc) { bus_dma_segment_t seg; bus_dmamap_t map; uint64_t addr; int error, rseg; int idx; /* * Allocate some DMA memory such that we have a "safe" target * address for MSIs. */ error = bus_dmamem_alloc(sc->sc_dmat, sizeof(uint32_t), sizeof(uint32_t), 0, &seg, 1, &rseg, BUS_DMA_WAITOK); if (error) return error; /* * Translate the CPU address into a bus address that we can * program into the hardware. */ error = bus_dmamap_create(sc->sc_dmat, sizeof(uint32_t), 1, sizeof(uint32_t), 0, BUS_DMA_WAITOK, &map); if (error) { bus_dmamem_free(sc->sc_dmat, &seg, 1); return error; } error = bus_dmamap_load_raw(sc->sc_dmat, map, &seg, 1, sizeof(uint32_t), BUS_DMA_WAITOK); if (error) { bus_dmamap_destroy(sc->sc_dmat, map); bus_dmamem_free(sc->sc_dmat, &seg, 1); return error; } addr = map->dm_segs[0].ds_addr; HWRITE4(sc, PCIE_MSI_ADDR_LO, addr); HWRITE4(sc, PCIE_MSI_ADDR_HI, addr >> 32); bus_dmamap_unload(sc->sc_dmat, map); bus_dmamap_destroy(sc->sc_dmat, map); /* * See if the device tree indicates that the hardware supports * more than 32 vectors. Some hardware supports more than 64, * but 64 is good enough for now. */ idx = OF_getindex(sc->sc_node, "msi1", "interrupt-names"); if (idx == -1) sc->sc_num_msi = 32; else sc->sc_num_msi = 64; KASSERT(sc->sc_num_msi <= DWPCIE_MAX_MSI); /* Enable, mask and clear all MSIs. */ for (idx = 0; idx < sc->sc_num_msi / 32; idx++) { HWRITE4(sc, PCIE_MSI_INTR_ENABLE(idx), 0xffffffff); HWRITE4(sc, PCIE_MSI_INTR_MASK(idx), 0xffffffff); HWRITE4(sc, PCIE_MSI_INTR_STATUS(idx), 0xffffffff); } idx = OF_getindex(sc->sc_node, "msi0", "interrupt-names"); if (idx == -1) idx = 0; sc->sc_msi_ih[0] = fdt_intr_establish_idx(sc->sc_node, idx, IPL_BIO | IPL_MPSAFE, dwpcie_msi0_intr, sc, sc->sc_dev.dv_xname); if (sc->sc_msi_ih[0] == NULL) { bus_dmamem_free(sc->sc_dmat, &seg, 1); return EINVAL; } idx = OF_getindex(sc->sc_node, "msi1", "interrupt-names"); if (idx == -1) goto finish; sc->sc_msi_ih[1] = fdt_intr_establish_idx(sc->sc_node, idx, IPL_BIO | IPL_MPSAFE, dwpcie_msi1_intr, sc, sc->sc_dev.dv_xname); if (sc->sc_msi_ih[1] == NULL) sc->sc_num_msi = 32; finish: /* * Hold on to the DMA memory such that nobody can use it to * actually do DMA transfers. */ sc->sc_msi_addr = addr; return 0; } int dwpcie_armada8k_init(struct dwpcie_softc *sc) { uint32_t reg; int timo; clock_enable_all(sc->sc_node); dwpcie_link_config(sc); if (!dwpcie_armada8k_link_up(sc)) { reg = HREAD4(sc, PCIE_GLOBAL_CTRL); reg &= ~PCIE_GLOBAL_CTRL_APP_LTSSM_EN; HWRITE4(sc, PCIE_GLOBAL_CTRL, reg); } /* * Setup Requester-ID to Stream-ID mapping * XXX: TF-A is supposed to set this up, but doesn't! */ HWRITE4(sc, PCIE_STREAMID, PCIE_STREAMID_8040); /* Enable Root Complex mode. */ reg = HREAD4(sc, PCIE_GLOBAL_CTRL); reg &= ~PCIE_GLOBAL_CTRL_DEVICE_TYPE_MASK; reg |= PCIE_GLOBAL_CTRL_DEVICE_TYPE_RC; HWRITE4(sc, PCIE_GLOBAL_CTRL, reg); HWRITE4(sc, PCIE_ARCACHE_TRC, PCIE_ARCACHE_TRC_DEFAULT); HWRITE4(sc, PCIE_AWCACHE_TRC, PCIE_AWCACHE_TRC_DEFAULT); reg = HREAD4(sc, PCIE_ARUSER); reg &= ~PCIE_AXUSER_DOMAIN_MASK; reg |= PCIE_AXUSER_DOMAIN_OUTER_SHARABLE; HWRITE4(sc, PCIE_ARUSER, reg); reg = HREAD4(sc, PCIE_AWUSER); reg &= ~PCIE_AXUSER_DOMAIN_MASK; reg |= PCIE_AXUSER_DOMAIN_OUTER_SHARABLE; HWRITE4(sc, PCIE_AWUSER, reg); if (!dwpcie_armada8k_link_up(sc)) { reg = HREAD4(sc, PCIE_GLOBAL_CTRL); reg |= PCIE_GLOBAL_CTRL_APP_LTSSM_EN; HWRITE4(sc, PCIE_GLOBAL_CTRL, reg); } for (timo = 40; timo > 0; timo--) { if (dwpcie_armada8k_link_up(sc)) break; delay(1000); } if (timo == 0) return ETIMEDOUT; sc->sc_ih = fdt_intr_establish(sc->sc_node, IPL_AUDIO | IPL_MPSAFE, dwpcie_armada8k_intr, sc, sc->sc_dev.dv_xname); /* Unmask INTx interrupts. */ HWRITE4(sc, PCIE_GLOBAL_INT_MASK, PCIE_GLOBAL_INT_MASK_INT_A | PCIE_GLOBAL_INT_MASK_INT_B | PCIE_GLOBAL_INT_MASK_INT_C | PCIE_GLOBAL_INT_MASK_INT_D); return 0; } int dwpcie_armada8k_link_up(struct dwpcie_softc *sc) { uint32_t reg, mask; mask = PCIE_GLOBAL_STATUS_RDLH_LINK_UP; mask |= PCIE_GLOBAL_STATUS_PHY_LINK_UP; reg = HREAD4(sc, PCIE_GLOBAL_STATUS); return ((reg & mask) == mask); } int dwpcie_armada8k_intr(void *arg) { struct dwpcie_softc *sc = arg; uint32_t cause; /* Acknowledge interrupts. */ cause = HREAD4(sc, PCIE_GLOBAL_INT_CAUSE); HWRITE4(sc, PCIE_GLOBAL_INT_CAUSE, cause); /* INTx interrupt, so not really ours. */ return 0; } int dwpcie_g12a_init(struct dwpcie_softc *sc) { uint32_t *reset_gpio; ssize_t reset_gpiolen; uint32_t reg; int error, timo; reset_gpiolen = OF_getproplen(sc->sc_node, "reset-gpios"); if (reset_gpiolen <= 0) return ENXIO; if (bus_space_map(sc->sc_iot, sc->sc_glue_base, sc->sc_glue_size, 0, &sc->sc_glue_ioh)) return ENOMEM; power_domain_enable(sc->sc_node); phy_enable(sc->sc_node, "pcie"); reset_assert_all(sc->sc_node); delay(500); reset_deassert_all(sc->sc_node); delay(500); clock_set_frequency(sc->sc_node, "port", 100000000UL); clock_enable_all(sc->sc_node); reset_gpio = malloc(reset_gpiolen, M_TEMP, M_WAITOK); OF_getpropintarray(sc->sc_node, "reset-gpios", reset_gpio, reset_gpiolen); gpio_controller_config_pin(reset_gpio, GPIO_CONFIG_OUTPUT); gpio_controller_set_pin(reset_gpio, 1); dwpcie_link_config(sc); reg = bus_space_read_4(sc->sc_iot, sc->sc_glue_ioh, PCIE_CFG0); reg |= PCIE_CFG0_APP_LTSSM_EN; bus_space_write_4(sc->sc_iot, sc->sc_glue_ioh, PCIE_CFG0, reg); gpio_controller_set_pin(reset_gpio, 1); delay(500); gpio_controller_set_pin(reset_gpio, 0); free(reset_gpio, M_TEMP, reset_gpiolen); for (timo = 40; timo > 0; timo--) { if (dwpcie_g12a_link_up(sc)) break; delay(1000); } if (timo == 0) return ETIMEDOUT; error = dwpcie_msi_init(sc); if (error) return error; return 0; } int dwpcie_g12a_link_up(struct dwpcie_softc *sc) { uint32_t reg; reg = bus_space_read_4(sc->sc_iot, sc->sc_glue_ioh, PCIE_STATUS12); if ((reg & PCIE_STATUS12_SMLH_LINK_UP) && (reg & PCIE_STATUS12_RDLH_LINK_UP) && (reg & PCIE_STATUS12_LTSSM_MASK) == PCIE_STATUS12_LTSSM_UP) return 1; return 0; } int dwpcie_imx8mq_init(struct dwpcie_softc *sc) { uint32_t *clkreq_gpio, *disable_gpio, *reset_gpio; ssize_t clkreq_gpiolen, disable_gpiolen, reset_gpiolen; struct regmap *anatop, *gpr, *phy; uint32_t off, reg; int error, timo; if (OF_is_compatible(sc->sc_node, "fsl,imx8mm-pcie")) { anatop = regmap_bycompatible("fsl,imx8mm-anatop"); gpr = regmap_bycompatible("fsl,imx8mm-iomuxc-gpr"); phy = regmap_bycompatible("fsl,imx7d-pcie-phy"); KASSERT(phy != NULL); } else { anatop = regmap_bycompatible("fsl,imx8mq-anatop"); gpr = regmap_bycompatible("fsl,imx8mq-iomuxc-gpr"); } KASSERT(anatop != NULL); KASSERT(gpr != NULL); clkreq_gpiolen = OF_getproplen(sc->sc_node, "clkreq-gpio"); disable_gpiolen = OF_getproplen(sc->sc_node, "disable-gpio"); reset_gpiolen = OF_getproplen(sc->sc_node, "reset-gpio"); if (clkreq_gpiolen > 0) { clkreq_gpio = malloc(clkreq_gpiolen, M_TEMP, M_WAITOK); OF_getpropintarray(sc->sc_node, "clkreq-gpio", clkreq_gpio, clkreq_gpiolen); gpio_controller_config_pin(clkreq_gpio, GPIO_CONFIG_OUTPUT); gpio_controller_set_pin(clkreq_gpio, 1); } if (disable_gpiolen > 0) { disable_gpio = malloc(disable_gpiolen, M_TEMP, M_WAITOK); OF_getpropintarray(sc->sc_node, "disable-gpio", disable_gpio, disable_gpiolen); gpio_controller_config_pin(disable_gpio, GPIO_CONFIG_OUTPUT); gpio_controller_set_pin(disable_gpio, 0); } if (reset_gpiolen > 0) { reset_gpio = malloc(reset_gpiolen, M_TEMP, M_WAITOK); OF_getpropintarray(sc->sc_node, "reset-gpio", reset_gpio, reset_gpiolen); gpio_controller_config_pin(reset_gpio, GPIO_CONFIG_OUTPUT); gpio_controller_set_pin(reset_gpio, 1); } power_domain_enable(sc->sc_node); reset_assert(sc->sc_node, "pciephy"); reset_assert(sc->sc_node, "apps"); reg = regmap_read_4(gpr, IOMUXC_GPR12); if (OF_getpropint(sc->sc_node, "ctrl-id", 0) == 0) { off = IOMUXC_GPR14; reg &= ~IMX8MQ_GPR_PCIE1_DEVICE_TYPE_MASK; reg |= IMX8MQ_GPR_PCIE1_DEVICE_TYPE_RC; } else { off = IOMUXC_GPR16; reg &= ~IMX8MQ_GPR_PCIE2_DEVICE_TYPE_MASK; reg |= IMX8MQ_GPR_PCIE2_DEVICE_TYPE_RC; } regmap_write_4(gpr, IOMUXC_GPR12, reg); if (OF_is_compatible(sc->sc_node, "fsl,imx8mm-pcie")) { if (OF_getproplen(sc->sc_node, "ext_osc") == 0 || OF_getpropint(sc->sc_node, "ext_osc", 0)) { reg = regmap_read_4(gpr, off); reg &= ~(IMX8MQ_GPR_PCIE_REF_USE_PAD | IMX8MM_GPR_PCIE_SSC_EN | IMX8MM_GPR_PCIE_POWER_OFF | IMX8MM_GPR_PCIE_REF_CLK_MASK); reg |= (IMX8MM_GPR_PCIE_AUX_EN | IMX8MM_GPR_PCIE_REF_CLK_EXT); regmap_write_4(gpr, off, reg); delay(100); reg = regmap_read_4(gpr, off); reg |= IMX8MM_GPR_PCIE_CMN_RST; regmap_write_4(gpr, off, reg); delay(200); } else { reg = regmap_read_4(gpr, off); reg &= ~(IMX8MQ_GPR_PCIE_REF_USE_PAD | IMX8MM_GPR_PCIE_SSC_EN | IMX8MM_GPR_PCIE_POWER_OFF | IMX8MM_GPR_PCIE_REF_CLK_MASK); reg |= (IMX8MM_GPR_PCIE_AUX_EN | IMX8MM_GPR_PCIE_REF_CLK_PLL); regmap_write_4(gpr, off, reg); delay(100); regmap_write_4(phy, IMX8MM_PCIE_PHY_CMN_REG62, IMX8MM_PCIE_PHY_CMN_REG62_PLL_CLK_OUT); regmap_write_4(phy, IMX8MM_PCIE_PHY_CMN_REG64, IMX8MM_PCIE_PHY_CMN_REG64_AUX_RX_TX_TERM); reg = regmap_read_4(gpr, off); reg |= IMX8MM_GPR_PCIE_CMN_RST; regmap_write_4(gpr, off, reg); delay(200); regmap_write_4(phy, IMX8MM_PCIE_PHY_TRSV_REG5, IMX8MM_PCIE_PHY_TRSV_REG5_GEN1_DEEMP); regmap_write_4(phy, IMX8MM_PCIE_PHY_TRSV_REG6, IMX8MM_PCIE_PHY_TRSV_REG6_GEN2_DEEMP); } } else { if (OF_getproplen(sc->sc_node, "ext_osc") == 0 || OF_getpropint(sc->sc_node, "ext_osc", 0)) { reg = regmap_read_4(gpr, off); reg |= IMX8MQ_GPR_PCIE_REF_USE_PAD; regmap_write_4(gpr, off, reg); } else { reg = regmap_read_4(gpr, off); reg &= ~IMX8MQ_GPR_PCIE_REF_USE_PAD; regmap_write_4(gpr, off, reg); regmap_write_4(anatop, ANATOP_PLLOUT_CTL, ANATOP_PLLOUT_CTL_CKE | ANATOP_PLLOUT_CTL_SEL_SYSPLL1); regmap_write_4(anatop, ANATOP_PLLOUT_DIV, ANATOP_PLLOUT_DIV_SYSPLL1); } } clock_enable(sc->sc_node, "pcie_phy"); clock_enable(sc->sc_node, "pcie_bus"); clock_enable(sc->sc_node, "pcie"); clock_enable(sc->sc_node, "pcie_aux"); /* Allow clocks to stabilize. */ delay(200); if (reset_gpiolen > 0) { gpio_controller_set_pin(reset_gpio, 1); delay(100000); gpio_controller_set_pin(reset_gpio, 0); } reset_deassert(sc->sc_node, "pciephy"); if (OF_is_compatible(sc->sc_node, "fsl,imx8mm-pcie")) { for (timo = 2000; timo > 0; timo--) { if (regmap_read_4(phy, IMX8MM_PCIE_PHY_CMN_REG75) == IMX8MM_PCIE_PHY_CMN_REG75_PLL_DONE) break; delay(10); } if (timo == 0) { error = ETIMEDOUT; goto err; } } reg = HREAD4(sc, 0x100000 + PCIE_RC_LCR); reg &= ~PCIE_RC_LCR_L1EL_MASK; reg |= PCIE_RC_LCR_L1EL_64US; HWRITE4(sc, 0x100000 + PCIE_RC_LCR, reg); dwpcie_link_config(sc); reg = HREAD4(sc, PCIE_RC_LCR); reg &= ~PCIE_RC_LCR_MAX_LINK_SPEEDS_MASK; reg |= PCIE_RC_LCR_MAX_LINK_SPEEDS_GEN1; HWRITE4(sc, PCIE_RC_LCR, reg); reset_deassert(sc->sc_node, "apps"); for (timo = 20000; timo > 0; timo--) { if (dwpcie_link_up(sc)) break; delay(10); } if (timo == 0) { error = ETIMEDOUT; goto err; } if (OF_getpropint(sc->sc_node, "fsl,max-link-speed", 1) >= 2) { reg = HREAD4(sc, PCIE_RC_LCR); reg &= ~PCIE_RC_LCR_MAX_LINK_SPEEDS_MASK; reg |= PCIE_RC_LCR_MAX_LINK_SPEEDS_GEN2; HWRITE4(sc, PCIE_RC_LCR, reg); reg = HREAD4(sc, PCIE_LINK_WIDTH_SPEED_CTRL); reg |= PCIE_LINK_WIDTH_SPEED_CTRL_CHANGE; HWRITE4(sc, PCIE_LINK_WIDTH_SPEED_CTRL, reg); for (timo = 20000; timo > 0; timo--) { if (dwpcie_link_up(sc)) break; delay(10); } if (timo == 0) { error = ETIMEDOUT; goto err; } } sc->sc_ih = fdt_intr_establish(sc->sc_node, IPL_AUDIO | IPL_MPSAFE, dwpcie_imx8mq_intr, sc, sc->sc_dev.dv_xname); /* Unmask INTx interrupts. */ HWRITE4(sc, PCIE_GLOBAL_INT_MASK, PCIE_GLOBAL_INT_MASK_INT_A | PCIE_GLOBAL_INT_MASK_INT_B | PCIE_GLOBAL_INT_MASK_INT_C | PCIE_GLOBAL_INT_MASK_INT_D); error = 0; err: if (clkreq_gpiolen > 0) free(clkreq_gpio, M_TEMP, clkreq_gpiolen); if (disable_gpiolen > 0) free(disable_gpio, M_TEMP, disable_gpiolen); if (reset_gpiolen > 0) free(reset_gpio, M_TEMP, reset_gpiolen); return error; } int dwpcie_imx8mq_intr(void *arg) { struct dwpcie_softc *sc = arg; uint32_t cause; /* Acknowledge interrupts. */ cause = HREAD4(sc, PCIE_GLOBAL_INT_CAUSE); HWRITE4(sc, PCIE_GLOBAL_INT_CAUSE, cause); /* INTx interrupt, so not really ours. */ return 0; } int dwpcie_fu740_init(struct dwpcie_softc *sc) { sc->sc_num_viewport = 8; return 0; } int dwpcie_rk3568_link_up(struct dwpcie_softc *sc) { uint32_t reg; reg = bus_space_read_4(sc->sc_iot, sc->sc_glue_ioh, PCIE_CLIENT_LTSSM_STATUS); if ((reg & PCIE_CLIENT_SMLH_LINK_UP) && (reg & PCIE_CLIENT_RDLH_LINK_UP) && (reg & PCIE_CLIENT_LTSSM_MASK) == PCIE_CLIENT_LTSSM_UP) return 1; return 0; } int dwpcie_rk3568_init(struct dwpcie_softc *sc) { uint32_t *reset_gpio; ssize_t reset_gpiolen; int error, idx, node; int pin, timo; sc->sc_num_viewport = 8; if (bus_space_map(sc->sc_iot, sc->sc_glue_base, sc->sc_glue_size, 0, &sc->sc_glue_ioh)) return ENOMEM; reset_assert_all(sc->sc_node); /* Power must be enabled before initializing the PHY. */ regulator_enable(OF_getpropint(sc->sc_node, "vpcie3v3-supply", 0)); phy_enable(sc->sc_node, "pcie-phy"); reset_deassert_all(sc->sc_node); clock_enable_all(sc->sc_node); if (dwpcie_rk3568_link_up(sc)) return 0; reset_gpiolen = OF_getproplen(sc->sc_node, "reset-gpios"); if (reset_gpiolen > 0) { reset_gpio = malloc(reset_gpiolen, M_TEMP, M_WAITOK); OF_getpropintarray(sc->sc_node, "reset-gpios", reset_gpio, reset_gpiolen); gpio_controller_config_pin(reset_gpio, GPIO_CONFIG_OUTPUT); gpio_controller_set_pin(reset_gpio, 1); } bus_space_write_4(sc->sc_iot, sc->sc_glue_ioh, PCIE_CLIENT_HOT_RESET_CTRL, PCIE_CLIENT_APP_LTSSM_ENABLE_ENHANCE); bus_space_write_4(sc->sc_iot, sc->sc_glue_ioh, PCIE_CLIENT_GENERAL_CON, PCIE_CLIENT_DEV_TYPE_RC); /* Assert PERST#. */ if (reset_gpiolen > 0) gpio_controller_set_pin(reset_gpio, 0); dwpcie_link_config(sc); /* Enable LTSSM. */ bus_space_write_4(sc->sc_iot, sc->sc_glue_ioh, PCIE_CLIENT_GENERAL_CON, PCIE_CLIENT_LINK_REQ_RST_GRT | PCIE_CLIENT_APP_LTSSM_ENABLE); /* * PERST# must remain asserted for at least 100us after the * reference clock becomes stable. But also has to remain * active at least 100ms after power up. Since we may have * just powered on the device, play it safe and use 100ms. */ delay(100000); /* Deassert PERST#. */ if (reset_gpiolen > 0) gpio_controller_set_pin(reset_gpio, 1); /* Wait for the link to come up. */ for (timo = 100; timo > 0; timo--) { if (dwpcie_rk3568_link_up(sc)) break; delay(10000); } if (timo == 0) { error = ETIMEDOUT; goto err; } node = OF_getnodebyname(sc->sc_node, "legacy-interrupt-controller"); idx = OF_getindex(sc->sc_node, "legacy", "interrupt-names"); if (node && idx != -1) { sc->sc_ih = fdt_intr_establish_idx(sc->sc_node, idx, IPL_BIO | IPL_MPSAFE, dwpcie_rk3568_intr, sc, sc->sc_dev.dv_xname); } if (sc->sc_ih) { for (pin = 0; pin < nitems(sc->sc_intx); pin++) TAILQ_INIT(&sc->sc_intx[pin]); sc->sc_ic.ic_node = node; sc->sc_ic.ic_cookie = sc; sc->sc_ic.ic_establish = dwpcie_rk3568_intr_establish; sc->sc_ic.ic_disestablish = dwpcie_rk3568_intr_disestablish; sc->sc_ic.ic_barrier = dwpcie_rk3568_intr_barrier; fdt_intr_register(&sc->sc_ic); } error = 0; err: if (reset_gpiolen > 0) free(reset_gpio, M_TEMP, reset_gpiolen); return error; } int dwpcie_rk3568_intr(void *arg) { struct dwpcie_softc *sc = arg; struct dwpcie_intx *di; uint32_t status; int pin, s; status = bus_space_read_4(sc->sc_iot, sc->sc_glue_ioh, PCIE_CLIENT_INTR_STATUS_LEGACY); for (pin = 0; pin < nitems(sc->sc_intx); pin++) { if ((status & (1 << pin)) == 0) continue; TAILQ_FOREACH(di, &sc->sc_intx[pin], di_next) { if ((di->di_flags & IPL_MPSAFE) == 0) KERNEL_LOCK(); s = splraise(di->di_ipl); if (di->di_func(di->di_arg)) di->di_count.ec_count++; splx(s); if ((di->di_flags & IPL_MPSAFE) == 0) KERNEL_UNLOCK(); } } return 1; } void * dwpcie_rk3568_intr_establish(void *cookie, int *cell, int level, struct cpu_info *ci, int (*func)(void *), void *arg, char *name) { struct dwpcie_softc *sc = (struct dwpcie_softc *)cookie; struct dwpcie_intx *di; int pin = cell[0]; uint32_t mask = (1U << pin); if (ci != NULL && !CPU_IS_PRIMARY(ci)) return NULL; if (pin < 0 || pin >= nitems(sc->sc_intx)) return NULL; /* Mask the interrupt. */ bus_space_write_4(sc->sc_iot, sc->sc_glue_ioh, PCIE_CLIENT_INTR_MASK_LEGACY, (mask << 16) | mask); intr_barrier(sc->sc_ih); di = malloc(sizeof(*di), M_DEVBUF, M_WAITOK | M_ZERO); di->di_func = func; di->di_arg = arg; di->di_ipl = level & IPL_IRQMASK; di->di_flags = level & IPL_FLAGMASK; di->di_pin = pin; di->di_name = name; if (name != NULL) evcount_attach(&di->di_count, name, &di->di_pin); di->di_sc = sc; TAILQ_INSERT_TAIL(&sc->sc_intx[pin], di, di_next); /* Unmask the interrupt. */ bus_space_write_4(sc->sc_iot, sc->sc_glue_ioh, PCIE_CLIENT_INTR_MASK_LEGACY, mask << 16); return di; } void dwpcie_rk3568_intr_disestablish(void *cookie) { struct dwpcie_intx *di = cookie; struct dwpcie_softc *sc = di->di_sc; uint32_t mask = (1U << di->di_pin); /* Mask the interrupt. */ bus_space_write_4(sc->sc_iot, sc->sc_glue_ioh, PCIE_CLIENT_INTR_MASK_LEGACY, (mask << 16) | mask); intr_barrier(sc->sc_ih); if (di->di_name) evcount_detach(&di->di_count); TAILQ_REMOVE(&sc->sc_intx[di->di_pin], di, di_next); if (!TAILQ_EMPTY(&sc->sc_intx[di->di_pin])) { /* Unmask the interrupt. */ bus_space_write_4(sc->sc_iot, sc->sc_glue_ioh, PCIE_CLIENT_INTR_MASK_LEGACY, mask << 16); } free(di, M_DEVBUF, sizeof(*di)); } void dwpcie_rk3568_intr_barrier(void *cookie) { struct dwpcie_intx *di = cookie; struct dwpcie_softc *sc = di->di_sc; intr_barrier(sc->sc_ih); } int dwpcie_sc8280xp_init(struct dwpcie_softc *sc) { sc->sc_num_viewport = 8; if (OF_getproplen(sc->sc_node, "msi-map") <= 0) return dwpcie_msi_init(sc); return 0; } void dwpcie_atu_write(struct dwpcie_softc *sc, int index, off_t reg, uint32_t val) { if (sc->sc_atu_unroll) { bus_space_write_4(sc->sc_iot, sc->sc_atu_ioh, IATU_OFFSET_UNROLL(index) + reg, val); return; } if (sc->sc_atu_viewport != index) { HWRITE4(sc, IATU_VIEWPORT, index); sc->sc_atu_viewport = index; } HWRITE4(sc, IATU_OFFSET_VIEWPORT + reg, val); } uint32_t dwpcie_atu_read(struct dwpcie_softc *sc, int index, off_t reg) { if (sc->sc_atu_unroll) { return bus_space_read_4(sc->sc_iot, sc->sc_atu_ioh, IATU_OFFSET_UNROLL(index) + reg); } if (sc->sc_atu_viewport != index) { HWRITE4(sc, IATU_VIEWPORT, index); sc->sc_atu_viewport = index; } return HREAD4(sc, IATU_OFFSET_VIEWPORT + reg); } void dwpcie_atu_disable(struct dwpcie_softc *sc, int index) { dwpcie_atu_write(sc, index, IATU_REGION_CTRL_2, 0); } void dwpcie_atu_config(struct dwpcie_softc *sc, int index, int type, uint64_t cpu_addr, uint64_t pci_addr, uint64_t size) { uint32_t reg; int timo; dwpcie_atu_write(sc, index, IATU_LWR_BASE_ADDR, cpu_addr); dwpcie_atu_write(sc, index, IATU_UPPER_BASE_ADDR, cpu_addr >> 32); dwpcie_atu_write(sc, index, IATU_LIMIT_ADDR, cpu_addr + size - 1); dwpcie_atu_write(sc, index, IATU_LWR_TARGET_ADDR, pci_addr); dwpcie_atu_write(sc, index, IATU_UPPER_TARGET_ADDR, pci_addr >> 32); dwpcie_atu_write(sc, index, IATU_REGION_CTRL_1, type); dwpcie_atu_write(sc, index, IATU_REGION_CTRL_2, IATU_REGION_CTRL_2_REGION_EN); for (timo = 5; timo > 0; timo--) { reg = dwpcie_atu_read(sc, index, IATU_REGION_CTRL_2); if (reg & IATU_REGION_CTRL_2_REGION_EN) break; delay(9000); } if (timo == 0) printf("%s:%d: timeout\n", __func__, __LINE__); } int dwpcie_link_up(struct dwpcie_softc *sc) { uint32_t reg; reg = HREAD4(sc, PCIE_PHY_DEBUG_R1); if ((reg & PCIE_PHY_DEBUG_R1_XMLH_LINK_UP) != 0 && (reg & PCIE_PHY_DEBUG_R1_XMLH_LINK_IN_TRAINING) == 0) return 1; return 0; } void dwpcie_attach_hook(struct device *parent, struct device *self, struct pcibus_attach_args *pba) { } int dwpcie_bus_maxdevs(void *v, int bus) { struct dwpcie_softc *sc = v; if (bus == sc->sc_bus || bus == sc->sc_bus + 1) return 1; return 32; } int dwpcie_find_node(int node, int bus, int device, int function) { uint32_t reg[5]; uint32_t phys_hi; int child; phys_hi = ((bus << 16) | (device << 11) | (function << 8)); for (child = OF_child(node); child; child = OF_peer(child)) { if (OF_getpropintarray(child, "reg", reg, sizeof(reg)) != sizeof(reg)) continue; if (reg[0] == phys_hi) return child; node = dwpcie_find_node(child, bus, device, function); if (node) return node; } return 0; } pcitag_t dwpcie_make_tag(void *v, int bus, int device, int function) { struct dwpcie_softc *sc = v; int node; node = dwpcie_find_node(sc->sc_node, bus, device, function); return (((pcitag_t)node << 32) | (bus << 24) | (device << 19) | (function << 16)); } void dwpcie_decompose_tag(void *v, pcitag_t tag, int *bp, int *dp, int *fp) { if (bp != NULL) *bp = (tag >> 24) & 0xff; if (dp != NULL) *dp = (tag >> 19) & 0x1f; if (fp != NULL) *fp = (tag >> 16) & 0x7; } int dwpcie_conf_size(void *v, pcitag_t tag) { return PCIE_CONFIG_SPACE_SIZE; } pcireg_t dwpcie_conf_read(void *v, pcitag_t tag, int reg) { struct dwpcie_softc *sc = v; int bus, dev, fn; uint32_t ret; dwpcie_decompose_tag(sc, tag, &bus, &dev, &fn); if (bus == sc->sc_bus) { KASSERT(dev == 0); tag = dwpcie_make_tag(sc, 0, dev, fn); return HREAD4(sc, PCITAG_OFFSET(tag) | reg); } if (bus == sc->sc_bus + 1) { dwpcie_atu_config(sc, IATU_VIEWPORT_INDEX1, IATU_REGION_CTRL_1_TYPE_CFG0, sc->sc_conf_base, PCITAG_OFFSET(tag), sc->sc_conf_size); } else { dwpcie_atu_config(sc, IATU_VIEWPORT_INDEX1, IATU_REGION_CTRL_1_TYPE_CFG1, sc->sc_conf_base, PCITAG_OFFSET(tag), sc->sc_conf_size); } ret = bus_space_read_4(sc->sc_iot, sc->sc_conf_ioh, reg); if (sc->sc_num_viewport <= 2 && sc->sc_io_size > 0) { dwpcie_atu_config(sc, IATU_VIEWPORT_INDEX1, IATU_REGION_CTRL_1_TYPE_IO, sc->sc_io_base, sc->sc_io_bus_addr, sc->sc_io_size); } return ret; } void dwpcie_conf_write(void *v, pcitag_t tag, int reg, pcireg_t data) { struct dwpcie_softc *sc = v; int bus, dev, fn; dwpcie_decompose_tag(sc, tag, &bus, &dev, &fn); if (bus == sc->sc_bus) { KASSERT(dev == 0); tag = dwpcie_make_tag(sc, 0, dev, fn); HWRITE4(sc, PCITAG_OFFSET(tag) | reg, data); return; } if (bus == sc->sc_bus + 1) { dwpcie_atu_config(sc, IATU_VIEWPORT_INDEX1, IATU_REGION_CTRL_1_TYPE_CFG0, sc->sc_conf_base, PCITAG_OFFSET(tag), sc->sc_conf_size); } else { dwpcie_atu_config(sc, IATU_VIEWPORT_INDEX1, IATU_REGION_CTRL_1_TYPE_CFG1, sc->sc_conf_base, PCITAG_OFFSET(tag), sc->sc_conf_size); } bus_space_write_4(sc->sc_iot, sc->sc_conf_ioh, reg, data); if (sc->sc_num_viewport <= 2 && sc->sc_io_size > 0) { dwpcie_atu_config(sc, IATU_VIEWPORT_INDEX1, IATU_REGION_CTRL_1_TYPE_IO, sc->sc_io_base, sc->sc_io_bus_addr, sc->sc_io_size); } } int dwpcie_probe_device_hook(void *v, struct pci_attach_args *pa) { struct dwpcie_softc *sc = v; uint16_t rid; int i; rid = pci_requester_id(pa->pa_pc, pa->pa_tag); pa->pa_dmat = iommu_device_map_pci(sc->sc_node, rid, pa->pa_dmat); for (i = 0; i < sc->sc_nranges; i++) { iommu_reserve_region_pci(sc->sc_node, rid, sc->sc_ranges[i].pci_base, sc->sc_ranges[i].size); } return 0; } int dwpcie_intr_map(struct pci_attach_args *pa, pci_intr_handle_t *ihp) { int pin = pa->pa_rawintrpin; if (pin == 0 || pin > PCI_INTERRUPT_PIN_MAX) return -1; if (pa->pa_tag == 0) return -1; ihp->ih_pc = pa->pa_pc; ihp->ih_tag = pa->pa_intrtag; ihp->ih_intrpin = pa->pa_intrpin; ihp->ih_type = PCI_INTX; return 0; } const char * dwpcie_intr_string(void *v, pci_intr_handle_t ih) { switch (ih.ih_type) { case PCI_MSI: return "msi"; case PCI_MSIX: return "msix"; } return "intx"; } struct dwpcie_msi * dwpcie_msi_establish(struct dwpcie_softc *sc, pci_intr_handle_t *ihp, int level, int (*func)(void *), void *arg, char *name) { pci_chipset_tag_t pc = ihp->ih_pc; pcitag_t tag = ihp->ih_tag; struct dwpcie_msi *dm; uint64_t msi_mask; int vec = ihp->ih_intrpin; int base, mme, nvec, off; pcireg_t reg; if (ihp->ih_type == PCI_MSI) { if (pci_get_capability(pc, tag, PCI_CAP_MSI, &off, ®) == 0) panic("%s: no msi capability", __func__); reg = pci_conf_read(ihp->ih_pc, ihp->ih_tag, off); mme = ((reg & PCI_MSI_MC_MME_MASK) >> PCI_MSI_MC_MME_SHIFT); if (vec >= (1 << mme)) return NULL; if (reg & PCI_MSI_MC_C64) base = pci_conf_read(pc, tag, off + PCI_MSI_MD64); else base = pci_conf_read(pc, tag, off + PCI_MSI_MD32); } else { mme = 0; base = 0; } if (vec == 0) { /* * Pre-allocate all the requested vectors. Remember * the number of requested vectors such that we can * deallocate them in one go. */ msi_mask = (1ULL << (1 << mme)) - 1; while (vec <= sc->sc_num_msi - (1 << mme)) { if ((sc->sc_msi_mask & (msi_mask << vec)) == 0) { sc->sc_msi_mask |= (msi_mask << vec); break; } vec += (1 << mme); } base = vec; nvec = (1 << mme); } else { KASSERT(ihp->ih_type == PCI_MSI); vec += base; nvec = 0; } if (vec >= sc->sc_num_msi) return NULL; if (ihp->ih_type == PCI_MSI) { if (reg & PCI_MSI_MC_C64) pci_conf_write(pc, tag, off + PCI_MSI_MD64, base); else pci_conf_write(pc, tag, off + PCI_MSI_MD32, base); } dm = &sc->sc_msi[vec]; KASSERT(dm->dm_func == NULL); dm->dm_func = func; dm->dm_arg = arg; dm->dm_ipl = level & IPL_IRQMASK; dm->dm_flags = level & IPL_FLAGMASK; dm->dm_vec = vec; dm->dm_nvec = nvec; dm->dm_name = name; if (name != NULL) evcount_attach(&dm->dm_count, name, &dm->dm_vec); /* Unmask the MSI. */ HCLR4(sc, PCIE_MSI_INTR_MASK(vec / 32), (1U << (vec % 32))); return dm; } void dwpcie_msi_disestablish(struct dwpcie_softc *sc, struct dwpcie_msi *dm) { uint64_t msi_mask = (1ULL << dm->dm_nvec) - 1; /* Mask the MSI. */ HSET4(sc, PCIE_MSI_INTR_MASK(dm->dm_vec / 32), (1U << (dm->dm_vec % 32))); if (dm->dm_name) evcount_detach(&dm->dm_count); dm->dm_func = NULL; /* * Unallocate all allocated vetcors if this is the first * vector for the device. */ sc->sc_msi_mask &= ~(msi_mask << dm->dm_vec); } void * dwpcie_intr_establish(void *v, pci_intr_handle_t ih, int level, struct cpu_info *ci, int (*func)(void *), void *arg, char *name) { struct dwpcie_softc *sc = v; struct dwpcie_intr_handle *pih; void *cookie = NULL; KASSERT(ih.ih_type != PCI_NONE); if (ih.ih_type != PCI_INTX) { struct dwpcie_msi *dm = NULL; bus_dma_tag_t dmat = ih.ih_dmat; bus_dma_segment_t seg; bus_dmamap_t map; uint64_t addr, data; if (sc->sc_msi_addr) { dm = dwpcie_msi_establish(sc, &ih, level, func, arg, name); if (dm == NULL) return NULL; addr = sc->sc_msi_addr; data = dm->dm_vec; } else { /* * Assume hardware passes Requester ID as * sideband data. */ addr = ih.ih_intrpin; data = pci_requester_id(ih.ih_pc, ih.ih_tag); cookie = fdt_intr_establish_msi_cpu(sc->sc_node, &addr, &data, level, ci, func, arg, (void *)name); if (cookie == NULL) return NULL; } pih = malloc(sizeof(*pih), M_DEVBUF, M_WAITOK | M_ZERO); pih->pih_ih.ih_ic = &dwpcie_ic; pih->pih_ih.ih_ih = cookie; pih->pih_sc = sc; pih->pih_dm = dm; if (sc->sc_msi_addr == 0) { if (bus_dmamap_create(dmat, sizeof(uint32_t), 1, sizeof(uint32_t), 0, BUS_DMA_WAITOK, &map)) { free(pih, M_DEVBUF, sizeof(*pih)); fdt_intr_disestablish(cookie); return NULL; } memset(&seg, 0, sizeof(seg)); seg.ds_addr = addr; seg.ds_len = sizeof(uint32_t); if (bus_dmamap_load_raw(dmat, map, &seg, 1, sizeof(uint32_t), BUS_DMA_WAITOK)) { bus_dmamap_destroy(dmat, map); free(pih, M_DEVBUF, sizeof(*pih)); fdt_intr_disestablish(cookie); return NULL; } addr = map->dm_segs[0].ds_addr; pih->pih_dmat = dmat; pih->pih_map = map; } if (ih.ih_type == PCI_MSIX) { pci_msix_enable(ih.ih_pc, ih.ih_tag, &sc->sc_bus_memt, ih.ih_intrpin, addr, data); } else pci_msi_enable(ih.ih_pc, ih.ih_tag, addr, data); } else { int bus, dev, fn; uint32_t reg[4]; dwpcie_decompose_tag(sc, ih.ih_tag, &bus, &dev, &fn); reg[0] = bus << 16 | dev << 11 | fn << 8; reg[1] = reg[2] = 0; reg[3] = ih.ih_intrpin; cookie = fdt_intr_establish_imap_cpu(sc->sc_node, reg, sizeof(reg), level, ci, func, arg, name); if (cookie == NULL) return NULL; pih = malloc(sizeof(*pih), M_DEVBUF, M_WAITOK | M_ZERO); pih->pih_ih.ih_ic = &dwpcie_ic; pih->pih_ih.ih_ih = cookie; } return pih; } void dwpcie_intr_disestablish(void *v, void *cookie) { struct dwpcie_intr_handle *pih = cookie; if (pih->pih_dm) dwpcie_msi_disestablish(pih->pih_sc, pih->pih_dm); else fdt_intr_disestablish(pih->pih_ih.ih_ih); if (pih->pih_dmat) { bus_dmamap_unload(pih->pih_dmat, pih->pih_map); bus_dmamap_destroy(pih->pih_dmat, pih->pih_map); } free(pih, M_DEVBUF, sizeof(*pih)); } int dwpcie_bs_iomap(bus_space_tag_t t, bus_addr_t addr, bus_size_t size, int flags, bus_space_handle_t *bshp) { struct dwpcie_softc *sc = t->bus_private; int i; for (i = 0; i < sc->sc_nranges; i++) { uint64_t pci_start = sc->sc_ranges[i].pci_base; uint64_t pci_end = pci_start + sc->sc_ranges[i].size; uint64_t phys_start = sc->sc_ranges[i].phys_base; if ((sc->sc_ranges[i].flags & 0x03000000) == 0x01000000 && addr >= pci_start && addr + size <= pci_end) { return bus_space_map(sc->sc_iot, addr - pci_start + phys_start, size, flags, bshp); } } return ENXIO; } int dwpcie_bs_memmap(bus_space_tag_t t, bus_addr_t addr, bus_size_t size, int flags, bus_space_handle_t *bshp) { struct dwpcie_softc *sc = t->bus_private; int i; for (i = 0; i < sc->sc_nranges; i++) { uint64_t pci_start = sc->sc_ranges[i].pci_base; uint64_t pci_end = pci_start + sc->sc_ranges[i].size; uint64_t phys_start = sc->sc_ranges[i].phys_base; if ((sc->sc_ranges[i].flags & 0x02000000) == 0x02000000 && addr >= pci_start && addr + size <= pci_end) { return bus_space_map(sc->sc_iot, addr - pci_start + phys_start, size, flags, bshp); } } return ENXIO; }