make SIOCGIFSFFPAGE support QSFP modules better.

basically the "read phy register" admin op uses the "address" field
on SFP modules for the i2c address, but uses that same field for
the "sff page number" for QSFP. this wraps the calls to the admin
op with sfp and qsfp specific code, and chooses between them depending
on what type of phy the fw reports.

tested by me and jmatthew@ pretty hard

1 files changed, 190 insertions, 58 deletions

diff --git a/sys/dev/pci/if_ixl.c b/sys/dev/pci/if_ixl.c
index 63b7aa22500..658a5711a0d 100644
--- a/sys/dev/pci/if_ixl.c
+++ b/sys/dev/pci/if_ixl.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_ixl.c,v 1.41 2019/07/29 05:00:13 jmatthew Exp $ */
+/*	$OpenBSD: if_ixl.c,v 1.42 2019/08/27 00:40:26 dlg Exp $ */
 
 /*
  * Copyright (c) 2013-2015, Intel Corporation
@@ -349,6 +349,10 @@ struct ixl_aq_phy_abilities {
 	uint8_t		phy_id[4];
 
 	uint8_t		module_type[3];
+#define IXL_SFF8024_ID_SFP		0x03
+#define IXL_SFF8024_ID_QSFP		0x0c
+#define IXL_SFF8024_ID_QSFP_PLUS	0x0d
+#define IXL_SFF8024_ID_QSFP28		0x11
 	uint8_t		qualified_module_count;
 #define IXL_AQ_PHY_MAX_QMS		16
 	struct ixl_aq_module_desc
@@ -604,7 +608,9 @@ struct ixl_aq_phy_reg_access {
 #define IXL_AQ_PHY_IF_EXTERNAL		1
 #define IXL_AQ_PHY_IF_MODULE		2
 	uint8_t		dev_addr;
-	uint16_t	_reserved1;
+	uint16_t	recall;
+#define IXL_AQ_PHY_QSFP_DEV_ADDR	0
+#define IXL_AQ_PHY_QSFP_LAST		1
 	uint32_t	reg;
 	uint32_t	val;
 	uint32_t	_reserved2;
@@ -1171,7 +1177,7 @@ static int	ixl_lldp_shut(struct ixl_softc *);
 static int	ixl_get_mac(struct ixl_softc *);
 static int	ixl_get_switch_config(struct ixl_softc *);
 static int	ixl_phy_mask_ints(struct ixl_softc *);
-static int	ixl_get_phy_abilities(struct ixl_softc *, uint64_t *);
+static int	ixl_get_phy_types(struct ixl_softc *, uint64_t *);
 static int	ixl_restart_an(struct ixl_softc *);
 static int	ixl_hmc(struct ixl_softc *);
 static void	ixl_hmc_free(struct ixl_softc *);
@@ -1192,10 +1198,8 @@ static void	ixl_hmc_pack(void *, const void *,
 static int	ixl_get_sffpage(struct ixl_softc *, struct if_sffpage *);
 static int	ixl_sff_get_byte(struct ixl_softc *, uint8_t, uint32_t,
 		    uint8_t *);
-#if 0
 static int	ixl_sff_set_byte(struct ixl_softc *, uint8_t, uint32_t,
 		    uint8_t);
-#endif
 
 static int	ixl_match(struct device *, void *, void *);
 static void	ixl_attach(struct device *, struct device *, void *);
@@ -1552,7 +1556,7 @@ ixl_attach(struct device *parent, struct device *self, void *aux)
 		goto free_hmc;
 	}
 
-	if (ixl_get_phy_abilities(sc, &phy_types) != 0) {
+	if (ixl_get_phy_types(sc, &phy_types) != 0) {
 		/* error printed by ixl_get_phy_abilities */
 		goto free_hmc;
 	}
@@ -3367,51 +3371,59 @@ ixl_phy_mask_ints(struct ixl_softc *sc)
 }
 
 static int
-ixl_get_phy_abilities(struct ixl_softc *sc, uint64_t *phy_types_ptr)
+ixl_get_phy_abilities(struct ixl_softc *sc,struct ixl_dmamem *idm)
 {
-	struct ixl_dmamem idm;
 	struct ixl_aq_desc iaq;
-	struct ixl_aq_phy_abilities *phy;
-	uint64_t phy_types;
 	int rv;
 
-	if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) {
-		printf("%s: unable to allocate switch config buffer\n",
-		    DEVNAME(sc));
-		return (-1);
-	}
-
 	memset(&iaq, 0, sizeof(iaq));
 	iaq.iaq_flags = htole16(IXL_AQ_BUF |
-	    (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0));
+	    (IXL_DMA_LEN(idm) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0));
 	iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_GET_ABILITIES);
-	iaq.iaq_datalen = htole16(IXL_AQ_BUFLEN);
+	htolem16(&iaq.iaq_datalen, IXL_DMA_LEN(idm));
 	iaq.iaq_param[0] = htole32(IXL_AQ_PHY_REPORT_INIT);
-	ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm));
+	ixl_aq_dva(&iaq, IXL_DMA_DVA(idm));
 
-	bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm),
+	bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm),
 	    BUS_DMASYNC_PREREAD);
 
 	rv = ixl_atq_poll(sc, &iaq, 250);
 
-	bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm),
+	bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm),
 	    BUS_DMASYNC_POSTREAD);
 
-	if (rv != 0) {
+	if (rv != 0)
+		return (-1);
+
+	return (lemtoh16(&iaq.iaq_retval));
+}
+
+static int
+ixl_get_phy_types(struct ixl_softc *sc, uint64_t *phy_types_ptr)
+{
+	struct ixl_dmamem idm;
+	struct ixl_aq_phy_abilities *phy;
+	uint64_t phy_types;
+	int rv;
+
+	if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) {
+		printf("%s: unable to allocate switch config buffer\n",
+		    DEVNAME(sc));
+		return (-1);
+	}
+
+	rv = ixl_get_phy_abilities(sc, &idm);
+	switch (rv) {
+	case -1:
 		printf("%s: GET PHY ABILITIES timeout\n", DEVNAME(sc));
-		rv = -1;
 		goto done;
-	}
-	switch (iaq.iaq_retval) {
-	case HTOLE16(IXL_AQ_RC_OK):
+	case IXL_AQ_RC_OK:
 		break;
-	case HTOLE16(IXL_AQ_RC_EIO):
+	case IXL_AQ_RC_EIO:
 		printf("%s: unable to query phy types\n", DEVNAME(sc));
-		rv = 0;
-		goto done;
+		break;
 	default:
-		printf("%s: GET PHY ABILITIIES error\n", DEVNAME(sc));
-		rv = -1;
+		printf("%s: GET PHY ABILITIIES error %u\n", DEVNAME(sc), rv);
 		goto done;
 	}
 
@@ -3429,6 +3441,32 @@ done:
 	return (rv);
 }
 
+/* this returns -1 on failure, or the sff module type */
+static int
+ixl_get_module_type(struct ixl_softc *sc)
+{
+	struct ixl_dmamem idm;
+	struct ixl_aq_phy_abilities *phy;
+	int rv;
+
+	if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0)
+		return (-1);
+
+	rv = ixl_get_phy_abilities(sc, &idm);
+	if (rv != IXL_AQ_RC_OK) {
+		rv = -1;
+		goto done;
+	}
+
+	phy = IXL_DMA_KVA(&idm);
+
+	rv = phy->module_type[0];
+
+done:
+	ixl_dmamem_free(sc, &idm);
+	return (rv);
+}
+
 static int
 ixl_get_link_status(struct ixl_softc *sc)
 {
@@ -3454,44 +3492,140 @@ ixl_get_link_status(struct ixl_softc *sc)
 	return (0);
 }
 
+struct ixl_sff_ops {
+	int (*open)(struct ixl_softc *sc, struct if_sffpage *, uint8_t *);
+	int (*get)(struct ixl_softc *sc, struct if_sffpage *, size_t);
+	int (*close)(struct ixl_softc *sc, struct if_sffpage *, uint8_t);
+};
+
+static int
+ixl_sfp_open(struct ixl_softc *sc, struct if_sffpage *sff, uint8_t *page)
+{
+	uint8_t npage;
+	int error;
+
+	if (sff->sff_addr != IFSFF_ADDR_EEPROM)
+		return (0);
+
+	error = ixl_sff_get_byte(sc, IFSFF_ADDR_EEPROM, 127, page);
+	if (error != 0)
+		return (error);
+	if (*page == sff->sff_page)
+		return (0);
+	error = ixl_sff_set_byte(sc, IFSFF_ADDR_EEPROM, 127, sff->sff_page);
+	if (error != 0)
+		return (error);
+
+	/* check if set worked */
+	error = ixl_sff_get_byte(sc, IFSFF_ADDR_EEPROM, 127, &npage);
+	if (error != 0)
+		return (error);
+	if (npage != sff->sff_page)
+		return (ENODEV);
+
+	return (0);
+}
+
+static int
+ixl_sfp_get(struct ixl_softc *sc, struct if_sffpage *sff, size_t i)
+{
+	return (ixl_sff_get_byte(sc, sff->sff_addr, i, &sff->sff_data[i]));
+}
+
+static int
+ixl_sfp_close(struct ixl_softc *sc, struct if_sffpage *sff, uint8_t page)
+{
+	int error;
+
+	if (sff->sff_addr != IFSFF_ADDR_EEPROM)
+		return (0);
+
+	if (page == sff->sff_page)
+		return (0);
+
+	error = ixl_sff_set_byte(sc, IFSFF_ADDR_EEPROM, 127, page);
+	if (error != 0)
+		return (error);
+
+	return (0);
+}
+
+static const struct ixl_sff_ops ixl_sfp_ops = {
+	ixl_sfp_open,
+	ixl_sfp_get,
+	ixl_sfp_close,
+};
+
+static int
+ixl_qsfp_open(struct ixl_softc *sc, struct if_sffpage *sff, uint8_t *page)
+{
+	uint8_t npage;
+	int error;
+
+	if (sff->sff_addr != IFSFF_ADDR_EEPROM)
+		return (EIO);
+
+	error = ixl_sff_get_byte(sc, sff->sff_page, 127, &npage);
+	if (error != 0)
+		return (error);
+	if (npage != sff->sff_page)
+		return (ENODEV);
+
+	return (0);
+}
+
+static int
+ixl_qsfp_get(struct ixl_softc *sc, struct if_sffpage *sff, size_t i)
+{
+	return (ixl_sff_get_byte(sc, sff->sff_page, i, &sff->sff_data[i]));
+}
+
+static int
+ixl_qsfp_close(struct ixl_softc *sc, struct if_sffpage *sff, uint8_t page)
+{
+	return (0);
+}
+
+static const struct ixl_sff_ops ixl_qsfp_ops = {
+	ixl_qsfp_open,
+	ixl_qsfp_get,
+	ixl_qsfp_close,
+};
+
 static int
 ixl_get_sffpage(struct ixl_softc *sc, struct if_sffpage *sff)
 {
-	uint8_t page = sff->sff_page;
+	const struct ixl_sff_ops *ops;
+	uint8_t page;
 	size_t i;
 	int error;
 
-#if 0
-	if (sff->sff_addr == IFSFF_ADDR_EEPROM) {
-		error = ixl_sff_get_byte(sc, IFSFF_ADDR_EEPROM, 127, &page);
-		if (error != 0)
-			return (error);
-		if (page != sff->sff_page) {
-			error = ixl_sff_set_byte(sc, IFSFF_ADDR_EEPROM, 127,
-			    sff->sff_page);
-			if (error != 0)
-				return (error);
-		}
+	switch (ixl_get_module_type(sc)) {
+	case -1:
+		return (EIO);
+	case IXL_SFF8024_ID_SFP:
+		ops = &ixl_sfp_ops;
+		break;
+	case IXL_SFF8024_ID_QSFP:
+	case IXL_SFF8024_ID_QSFP_PLUS:
+	case IXL_SFF8024_ID_QSFP28:
+		ops = &ixl_qsfp_ops;
+		break;
+	default:
+		return (EOPNOTSUPP);
 	}
-#endif
+
+	error = (*ops->open)(sc, sff, &page);
+	if (error != 0)
+		return (error);
 
 	for (i = 0; i < sizeof(sff->sff_data); i++) {
-		error = ixl_sff_get_byte(sc, page, i,
-		    &sff->sff_data[i]);
+		error = (*ops->get)(sc, sff, i);
 		if (error != 0)
 			return (error);
 	}
 
-#if 0
-	if (sff->sff_addr == IFSFF_ADDR_EEPROM) {
-		if (page != sff->sff_page) {
-			error = ixl_sff_set_byte(sc, IFSFF_ADDR_EEPROM, 127,
-			    page);
-			if (error != 0)
-				return (error);
-		}
-	}
-#endif
+	error = (*ops->close)(sc, sff, page);
 
 	return (0);
 }
@@ -3537,7 +3671,6 @@ ixl_sff_get_byte(struct ixl_softc *sc, uint8_t dev, uint32_t reg, uint8_t *p)
 	return (0);
 }
 
-#if 0
 static int
 ixl_sff_set_byte(struct ixl_softc *sc, uint8_t dev, uint32_t reg, uint8_t v)
 {
@@ -3577,7 +3710,6 @@ ixl_sff_set_byte(struct ixl_softc *sc, uint8_t dev, uint32_t reg, uint8_t v)
 
 	return (0);
 }
-#endif
 
 static int
 ixl_get_vsi(struct ixl_softc *sc)