Add mcx(4), a driver for Mellanox ConnectX-4 (and later) Ethernet

controllers, written by dlg@ and me based on the cut down non-NDA
programming reference manual Mellanox were nice enough to release.

ok dlg@ deraadt@

2 files changed, 5473 insertions, 1 deletions

diff --git a/sys/dev/pci/files.pci b/sys/dev/pci/files.pci
index c29e90cfd6d..6ca7e0218d1 100644
--- a/sys/dev/pci/files.pci
+++ b/sys/dev/pci/files.pci
@@ -1,4 +1,4 @@
-#	$OpenBSD: files.pci,v 1.335 2018/07/09 10:46:15 jmatthew Exp $
+#	$OpenBSD: files.pci,v 1.336 2019/05/04 06:40:33 jmatthew Exp $
 #	$NetBSD: files.pci,v 1.20 1996/09/24 17:47:15 christos Exp $
 #
 # Config file and device description for machine-independent PCI code.
@@ -828,5 +828,10 @@ device	bnxt: ether, ifnet, ifmedia
 attach	bnxt at pci
 file	dev/pci/if_bnxt.c		bnxt
 
+# Mellanox ConnectX-4 and later
+device  mcx: ether, ifnet, ifmedia
+attach  mcx at pci
+file    dev/pci/if_mcx.c                mcx
+
 include "dev/pci/files.agp"
 include "dev/pci/drm/files.drm"
diff --git a/sys/dev/pci/if_mcx.c b/sys/dev/pci/if_mcx.c
new file mode 100644
index 00000000000..ad5516479aa
--- /dev/null
+++ b/sys/dev/pci/if_mcx.c
@@ -0,0 +1,5467 @@
+/*	$OpenBSD: if_mcx.c,v 1.1 2019/05/04 06:40:33 jmatthew Exp $ */
+
+/*
+ * Copyright (c) 2017 David Gwynne <dlg@openbsd.org>
+ * Copyright (c) 2019 Jonathan Matthew <jmatthew@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "bpfilter.h"
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/sockio.h>
+#include <sys/mbuf.h>
+#include <sys/kernel.h>
+#include <sys/socket.h>
+#include <sys/device.h>
+#include <sys/pool.h>
+#include <sys/queue.h>
+#include <sys/timeout.h>
+#include <sys/task.h>
+
+#include <machine/bus.h>
+#include <machine/intr.h>
+
+#include <net/if.h>
+#include <net/if_dl.h>
+#include <net/if_media.h>
+
+#if NBPFILTER > 0
+#include <net/bpf.h>
+#endif
+
+#include <netinet/in.h>
+#include <netinet/if_ether.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+#include <dev/pci/pcidevs.h>
+
+#define BUS_DMASYNC_PRERW	(BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)
+#define BUS_DMASYNC_POSTRW	(BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE)
+
+#define MCX_HCA_BAR	PCI_MAPREG_START /* BAR 0 */
+
+#define MCX_FW_VER	 	  0x0000
+#define  MCX_FW_VER_MAJOR(_v)		((_v) & 0xffff)
+#define  MCX_FW_VER_MINOR(_v)		((_v) >> 16)
+#define MCX_CMDIF_FW_SUBVER	  0x0004
+#define  MCX_FW_VER_SUBMINOR(_v)	((_v) & 0xffff)
+#define  MCX_CMDIF(_v)			((_v) >> 16)
+
+#define MCX_ISSI		 1 /* as per the PRM */
+#define MCX_CMD_IF_SUPPORTED	 5
+
+#define MCX_HARDMTU		 9500
+
+#define MCX_MAX_CQS		 2		/* rq, sq */
+
+/* queue sizes */
+#define MCX_LOG_EQ_SIZE		 6		/* one page */
+#define MCX_LOG_CQ_SIZE		 10
+#define MCX_LOG_RQ_SIZE		 5
+#define MCX_LOG_SQ_SIZE		 10
+
+/* doorbell offsets */
+#define MCX_CQ_DOORBELL_OFFSET	 0
+#define MCX_CQ_DOORBELL_SIZE	 16
+#define MCX_RQ_DOORBELL_OFFSET	 64
+#define MCX_SQ_DOORBELL_OFFSET	 64
+
+/* uar registers */
+#define MCX_UAR_CQ_DOORBELL	 0x20
+#define MCX_UAR_EQ_DOORBELL_ARM	 0x40
+#define MCX_UAR_EQ_DOORBELL	 0x48
+#define MCX_UAR_BF		 0x800
+
+#define MCX_CMDQ_ADDR_HI		 0x0010
+#define MCX_CMDQ_ADDR_LO		 0x0014
+#define MCX_CMDQ_ADDR_NMASK		0xfff
+#define MCX_CMDQ_LOG_SIZE(_v)		((_v) >> 4 & 0xf)
+#define MCX_CMDQ_LOG_STRIDE(_v)		((_v) >> 0 & 0xf)
+#define MCX_CMDQ_INTERFACE_MASK		(0x3 << 8)
+#define MCX_CMDQ_INTERFACE_FULL_DRIVER	(0x0 << 8)
+#define MCX_CMDQ_INTERFACE_DISABLED	(0x1 << 8)
+
+#define MCX_CMDQ_DOORBELL		0x0018
+
+#define MCX_STATE		0x01fc
+#define MCX_STATE_MASK			(1 << 31)
+#define MCX_STATE_INITIALIZING		(1 << 31)
+#define MCX_STATE_READY			(0 << 31)
+#define MCX_STATE_INTERFACE_MASK	(0x3 << 24)
+#define MCX_STATE_INTERFACE_FULL_DRIVER	(0x0 << 24)
+#define MCX_STATE_INTERFACE_DISABLED	(0x1 << 24)
+
+#define MCX_INTERNAL_TIMER	0x1000
+#define MCX_INTERNAL_TIMER_H	0x1000
+#define MCX_INTERNAL_TIMER_L	0x1004
+
+#define MCX_CLEAR_INT		0x100c
+
+#define MCX_REG_OP_WRITE	0
+#define MCX_REG_OP_READ		1
+
+#define MCX_REG_PMTU		0x5003
+#define MCX_REG_PTYS		0x5004
+#define MCX_REG_PAOS		0x5006
+#define MCX_REG_PFCC		0x5007
+#define MCX_REG_PPCNT		0x5008
+
+#define MCX_ETHER_CAP_SGMII	(1 << 0)
+#define MCX_ETHER_CAP_1000_KX	(1 << 1)
+#define MCX_ETHER_CAP_10G_CX4	(1 << 2)
+#define MCX_ETHER_CAP_10G_KX4	(1 << 3)
+#define MCX_ETHER_CAP_10G_KR	(1 << 4)
+#define MCX_ETHER_CAP_40G_CR4	(1 << 6)
+#define MCX_ETHER_CAP_40G_KR4	(1 << 7)
+#define MCX_ETHER_CAP_10G_CR	(1 << 12)
+#define MCX_ETHER_CAP_10G_SR	(1 << 13)
+#define MCX_ETHER_CAP_10G_LR	(1 << 14)
+#define MCX_ETHER_CAP_40G_SR4	(1 << 15)
+#define MCX_ETHER_CAP_40G_LR4	(1 << 16)
+#define MCX_ETHER_CAP_50G_SR2	(1 << 18)
+#define MCX_ETHER_CAP_100G_CR4	(1 << 20)
+#define MCX_ETHER_CAP_100G_SR4	(1 << 21)
+#define MCX_ETHER_CAP_100G_KR4	(1 << 22)
+#define MCX_ETHER_CAP_25G_CR	(1 << 27)
+#define MCX_ETHER_CAP_25G_KR	(1 << 28)
+#define MCX_ETHER_CAP_25G_SR	(1 << 29)
+#define MCX_ETHER_CAP_50G_CR2	(1 << 30)
+#define MCX_ETHER_CAP_50G_KR2	(1 << 31)
+
+#define MCX_PAGE_SHIFT		12
+#define MCX_PAGE_SIZE		(1 << MCX_PAGE_SHIFT)
+#define MCX_MAX_CQE		32
+
+#define MCX_CMD_QUERY_HCA_CAP	0x100
+#define MCX_CMD_QUERY_ADAPTER	0x101
+#define MCX_CMD_INIT_HCA	0x102
+#define MCX_CMD_TEARDOWN_HCA	0x103
+#define MCX_CMD_ENABLE_HCA	0x104
+#define MCX_CMD_DISABLE_HCA	0x105
+#define MCX_CMD_QUERY_PAGES	0x107
+#define MCX_CMD_MANAGE_PAGES	0x108
+#define MCX_CMD_SET_HCA_CAP	0x109
+#define MCX_CMD_QUERY_ISSI	0x10a
+#define MCX_CMD_SET_ISSI	0x10b
+#define MCX_CMD_SET_DRIVER_VERSION \
+				0x10d
+#define MCX_CMD_QUERY_SPECIAL_CONTEXTS \
+				0x203
+#define MCX_CMD_CREATE_EQ	0x301
+#define MCX_CMD_DESTROY_EQ	0x302
+#define MCX_CMD_CREATE_CQ	0x400
+#define MCX_CMD_DESTROY_CQ	0x401
+#define MCX_CMD_QUERY_NIC_VPORT_CONTEXT \
+				0x754
+#define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT \
+				0x755
+#define MCX_CMD_QUERY_VPORT_COUNTERS \
+				0x770
+#define MCX_CMD_ALLOC_PD	0x800
+#define MCX_CMD_ALLOC_UAR	0x802
+#define MCX_CMD_ACCESS_REG	0x805
+#define MCX_CMD_ALLOC_TRANSPORT_DOMAIN \
+				0x816
+#define MCX_CMD_CREATE_TIR	0x900
+#define MCX_CMD_CREATE_SQ	0x904
+#define MCX_CMD_MODIFY_SQ	0x905
+#define MCX_CMD_QUERY_SQ	0x907
+#define MCX_CMD_CREATE_RQ	0x908
+#define MCX_CMD_MODIFY_RQ	0x909
+#define MCX_CMD_QUERY_RQ	0x90b
+#define MCX_CMD_CREATE_TIS	0x912
+#define MCX_CMD_SET_FLOW_TABLE_ROOT \
+				0x92f
+#define MCX_CMD_CREATE_FLOW_TABLE \
+				0x930
+#define MCX_CMD_QUERY_FLOW_TABLE \
+				0x932
+#define MCX_CMD_CREATE_FLOW_GROUP \
+				0x933
+#define MCX_CMD_QUERY_FLOW_GROUP \
+				0x935
+#define MCX_CMD_SET_FLOW_TABLE_ENTRY \
+				0x936
+#define MCX_CMD_QUERY_FLOW_TABLE_ENTRY \
+				0x937
+#define MCX_CMD_ALLOC_FLOW_COUNTER \
+				0x939
+#define MCX_CMD_QUERY_FLOW_COUNTER \
+				0x93b
+
+#define MCX_QUEUE_STATE_RST	0
+#define MCX_QUEUE_STATE_RDY	1
+#define MCX_QUEUE_STATE_ERR	3
+
+#define MCX_FLOW_TABLE_TYPE_RX	0
+#define MCX_FLOW_TABLE_TYPE_TX	1
+
+#define MCX_CMDQ_INLINE_DATASIZE 16
+
+struct mcx_cmdq_entry {
+	uint8_t			cq_type;
+#define MCX_CMDQ_TYPE_PCIE		0x7
+	uint8_t			cq_reserved0[3];
+
+	uint32_t		cq_input_length;
+	uint64_t		cq_input_ptr;
+	uint8_t			cq_input_data[MCX_CMDQ_INLINE_DATASIZE];
+
+	uint8_t			cq_output_data[MCX_CMDQ_INLINE_DATASIZE];
+	uint64_t		cq_output_ptr;
+	uint32_t		cq_output_length;
+
+	uint8_t			cq_token;
+	uint8_t			cq_signature;
+	uint8_t			cq_reserved1[1];
+	uint8_t			cq_status;
+#define MCX_CQ_STATUS_SHIFT		1
+#define MCX_CQ_STATUS_MASK		(0x7f << MCX_CQ_STATUS_SHIFT)
+#define MCX_CQ_STATUS_OK		(0x00 << MCX_CQ_STATUS_SHIFT)
+#define MCX_CQ_STATUS_INT_ERR		(0x01 << MCX_CQ_STATUS_SHIFT)
+#define MCX_CQ_STATUS_BAD_OPCODE	(0x02 << MCX_CQ_STATUS_SHIFT)
+#define MCX_CQ_STATUS_BAD_PARAM		(0x03 << MCX_CQ_STATUS_SHIFT)
+#define MCX_CQ_STATUS_BAD_SYS_STATE	(0x04 << MCX_CQ_STATUS_SHIFT)
+#define MCX_CQ_STATUS_BAD_RESOURCE	(0x05 << MCX_CQ_STATUS_SHIFT)
+#define MCX_CQ_STATUS_RESOURCE_BUSY	(0x06 << MCX_CQ_STATUS_SHIFT)
+#define MCX_CQ_STATUS_EXCEED_LIM	(0x08 << MCX_CQ_STATUS_SHIFT)
+#define MCX_CQ_STATUS_BAD_RES_STATE	(0x09 << MCX_CQ_STATUS_SHIFT)
+#define MCX_CQ_STATUS_BAD_INDEX		(0x0a << MCX_CQ_STATUS_SHIFT)
+#define MCX_CQ_STATUS_NO_RESOURCES	(0x0f << MCX_CQ_STATUS_SHIFT)
+#define MCX_CQ_STATUS_BAD_INPUT_LEN	(0x50 << MCX_CQ_STATUS_SHIFT)
+#define MCX_CQ_STATUS_BAD_OUTPUT_LEN	(0x51 << MCX_CQ_STATUS_SHIFT)
+#define MCX_CQ_STATUS_BAD_RESOURCE_STATE \
+					(0x10 << MCX_CQ_STATUS_SHIFT)
+#define MCX_CQ_STATUS_BAD_SIZE		(0x40 << MCX_CQ_STATUS_SHIFT)
+#define MCX_CQ_STATUS_OWN_MASK		0x1
+#define MCX_CQ_STATUS_OWN_SW		0x0
+#define MCX_CQ_STATUS_OWN_HW		0x1
+} __packed __aligned(8);
+
+#define MCX_CMDQ_MAILBOX_DATASIZE	512
+
+struct mcx_cmdq_mailbox {
+	uint8_t			mb_data[MCX_CMDQ_MAILBOX_DATASIZE];
+	uint8_t			mb_reserved0[48];
+	uint64_t		mb_next_ptr;
+	uint32_t		mb_block_number;
+	uint8_t			mb_reserved1[1];
+	uint8_t			mb_token;
+	uint8_t			mb_ctrl_signature;
+	uint8_t			mb_signature;
+} __packed __aligned(8);
+
+#define MCX_CMDQ_MAILBOX_ALIGN	(1 << 10)
+#define MCX_CMDQ_MAILBOX_SIZE	roundup(sizeof(struct mcx_cmdq_mailbox), \
+				    MCX_CMDQ_MAILBOX_ALIGN)
+/*
+ * command mailbox structres
+ */
+
+struct mcx_cmd_enable_hca_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[2];
+	uint16_t		cmd_function_id;
+	uint8_t			cmd_reserved2[4];
+} __packed __aligned(4);
+
+struct mcx_cmd_enable_hca_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint8_t			cmd_reserved1[4];
+} __packed __aligned(4);
+
+struct mcx_cmd_init_hca_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_init_hca_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_teardown_hca_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[2];
+#define MCX_CMD_TEARDOWN_HCA_GRACEFUL	0x0
+#define MCX_CMD_TEARDOWN_HCA_PANIC	0x1
+	uint16_t		cmd_profile;
+	uint8_t			cmd_reserved2[4];
+} __packed __aligned(4);
+
+struct mcx_cmd_teardown_hca_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_access_reg_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[2];
+	uint16_t		cmd_register_id;
+	uint32_t		cmd_argument;
+} __packed __aligned(4);
+
+struct mcx_cmd_access_reg_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_reg_pmtu {
+	uint8_t			rp_reserved1;
+	uint8_t			rp_local_port;
+	uint8_t			rp_reserved2[2];
+	uint16_t		rp_max_mtu;
+	uint8_t			rp_reserved3[2];
+	uint16_t		rp_admin_mtu;
+	uint8_t			rp_reserved4[2];
+	uint16_t		rp_oper_mtu;
+	uint8_t			rp_reserved5[2];
+} __packed __aligned(4);
+
+struct mcx_reg_ptys {
+	uint8_t			rp_reserved1;
+	uint8_t			rp_local_port;
+	uint8_t			rp_reserved2;
+	uint8_t			rp_proto_mask;
+#define MCX_REG_PTYS_PROTO_MASK_ETH		(1 << 2)
+	uint8_t			rp_reserved3[8];
+	uint32_t		rp_eth_proto_cap;
+	uint8_t			rp_reserved4[8];
+	uint32_t		rp_eth_proto_admin;
+	uint8_t			rp_reserved5[8];
+	uint32_t		rp_eth_proto_oper;
+	uint8_t			rp_reserved6[24];
+} __packed __aligned(4);
+
+struct mcx_reg_paos {
+	uint8_t			rp_reserved1;
+	uint8_t			rp_local_port;
+	uint8_t			rp_admin_status;
+	uint8_t			rp_oper_status;
+	uint8_t			rp_admin_state_update;
+#define MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN	(1 << 7)
+	uint8_t			rp_reserved2[11];
+} __packed __aligned(4);
+
+struct mcx_reg_pfcc {
+	uint8_t			rp_reserved1;
+	uint8_t			rp_local_port;
+	uint8_t			rp_reserved2[3];
+	uint8_t			rp_prio_mask_tx;
+	uint8_t			rp_reserved3;
+	uint8_t			rp_prio_mask_rx;
+	uint8_t			rp_pptx_aptx;
+	uint8_t			rp_pfctx;
+	uint8_t			rp_fctx_dis;
+	uint8_t			rp_reserved4;
+	uint8_t			rp_pprx_aprx;
+	uint8_t			rp_pfcrx;
+	uint8_t			rp_reserved5[2];
+	uint16_t		rp_dev_stall_min;
+	uint16_t		rp_dev_stall_crit;
+	uint8_t			rp_reserved6[12];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_issi_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_issi_il_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint8_t			cmd_reserved1[2];
+	uint16_t		cmd_current_issi;
+	uint8_t			cmd_reserved2[4];
+} __packed __aligned(4);
+
+CTASSERT(sizeof(struct mcx_cmd_query_issi_il_out) == MCX_CMDQ_INLINE_DATASIZE);
+
+struct mcx_cmd_query_issi_mb_out {
+	uint8_t			cmd_reserved2[16];
+	uint8_t			cmd_supported_issi[80]; /* very big endian */
+} __packed __aligned(4);
+
+CTASSERT(sizeof(struct mcx_cmd_query_issi_mb_out) <= MCX_CMDQ_MAILBOX_DATASIZE);
+
+struct mcx_cmd_set_issi_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[2];
+	uint16_t		cmd_current_issi;
+	uint8_t			cmd_reserved2[4];
+} __packed __aligned(4);
+
+CTASSERT(sizeof(struct mcx_cmd_set_issi_in) <= MCX_CMDQ_INLINE_DATASIZE);
+
+struct mcx_cmd_set_issi_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+CTASSERT(sizeof(struct mcx_cmd_set_issi_out) <= MCX_CMDQ_INLINE_DATASIZE);
+
+struct mcx_cmd_query_pages_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+#define MCX_CMD_QUERY_PAGES_BOOT	0x01
+#define MCX_CMD_QUERY_PAGES_INIT	0x02
+#define MCX_CMD_QUERY_PAGES_REGULAR	0x03
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_pages_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint8_t			cmd_reserved1[2];
+	uint16_t		cmd_func_id;
+	uint32_t		cmd_num_pages;
+} __packed __aligned(4);
+
+struct mcx_cmd_manage_pages_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+#define MCX_CMD_MANAGE_PAGES_ALLOC_FAIL \
+					0x00
+#define MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS \
+					0x01
+#define MCX_CMD_MANAGE_PAGES_HCA_RETURN_PAGES \
+					0x02
+	uint8_t			cmd_reserved1[2];
+	uint16_t		cmd_func_id;
+	uint32_t		cmd_input_num_entries;
+} __packed __aligned(4);
+
+CTASSERT(sizeof(struct mcx_cmd_manage_pages_in) == MCX_CMDQ_INLINE_DATASIZE);
+
+struct mcx_cmd_manage_pages_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint32_t		cmd_output_num_entries;
+	uint8_t			cmd_reserved1[4];
+} __packed __aligned(4);
+
+CTASSERT(sizeof(struct mcx_cmd_manage_pages_out) == MCX_CMDQ_INLINE_DATASIZE);
+
+struct mcx_cmd_query_hca_cap_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+#define MCX_CMD_QUERY_HCA_CAP_MAX	(0x0 << 0)
+#define MCX_CMD_QUERY_HCA_CAP_CURRENT	(0x1 << 0)
+#define MCX_CMD_QUERY_HCA_CAP_DEVICE	(0x0 << 1)
+#define MCX_CMD_QUERY_HCA_CAP_OFFLOAD	(0x1 << 1)
+#define MCX_CMD_QUERY_HCA_CAP_FLOW	(0x7 << 1)
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_hca_cap_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+#define MCX_HCA_CAP_LEN			0x1000
+#define MCX_HCA_CAP_NMAILBOXES		\
+	(MCX_HCA_CAP_LEN / MCX_CMDQ_MAILBOX_DATASIZE)
+
+#if __GNUC_PREREQ__(4, 3)
+#define __counter__		__COUNTER__
+#else
+#define __counter__		__LINE__
+#endif
+
+#define __token(_tok, _num)	_tok##_num
+#define _token(_tok, _num)	__token(_tok, _num)
+#define __reserved__		_token(__reserved, __counter__)
+
+struct mcx_cap_device {
+	uint8_t			reserved0[16];
+
+	uint8_t			log_max_srq_sz;
+	uint8_t			log_max_qp_sz;
+	uint8_t			__reserved__[1];
+	uint8_t			log_max_qp; /* 5 bits */
+#define MCX_CAP_DEVICE_LOG_MAX_QP	0x1f
+
+	uint8_t			__reserved__[1];
+	uint8_t			log_max_srq; /* 5 bits */
+#define MCX_CAP_DEVICE_LOG_MAX_SRQ	0x1f
+	uint8_t			__reserved__[2];
+
+	uint8_t			__reserved__[1];
+	uint8_t			log_max_cq_sz;
+	uint8_t			__reserved__[1];
+	uint8_t			log_max_cq; /* 5 bits */
+#define MCX_CAP_DEVICE_LOG_MAX_CQ	0x1f
+
+	uint8_t			log_max_eq_sz;
+	uint8_t			log_max_mkey; /* 6 bits */
+#define MCX_CAP_DEVICE_LOG_MAX_MKEY	0x3f
+	uint8_t			__reserved__[1];
+	uint8_t			log_max_eq; /* 4 bits */
+#define MCX_CAP_DEVICE_LOG_MAX_EQ	0x0f
+
+	uint8_t			max_indirection;
+	uint8_t			log_max_mrw_sz; /* 7 bits */
+#define MCX_CAP_DEVICE_LOG_MAX_MRW_SZ	0x7f
+	uint8_t			teardown_log_max_msf_list_size;
+#define MCX_CAP_DEVICE_FORCE_TEARDOWN	0x80
+#define MCX_CAP_DEVICE_LOG_MAX_MSF_LIST_SIZE \
+					0x3f
+	uint8_t			log_max_klm_list_size; /* 6 bits */
+#define MCX_CAP_DEVICE_LOG_MAX_KLM_LIST_SIZE \
+					0x3f
+
+	uint8_t			__reserved__[1];
+	uint8_t			log_max_ra_req_dc; /* 6 bits */
+#define MCX_CAP_DEVICE_LOG_MAX_REQ_DC	0x3f
+	uint8_t			__reserved__[1];
+	uint8_t			log_max_ra_res_dc; /* 6 bits */
+#define MCX_CAP_DEVICE_LOG_MAX_RA_RES_DC \
+					0x3f
+
+	uint8_t			__reserved__[1];
+	uint8_t			log_max_ra_req_qp; /* 6 bits */
+#define MCX_CAP_DEVICE_LOG_MAX_RA_REQ_QP \
+					0x3f
+	uint8_t			__reserved__[1];
+	uint8_t			log_max_ra_res_qp; /* 6 bits */
+#define MCX_CAP_DEVICE_LOG_MAX_RA_RES_QP \
+					0x3f
+
+	uint8_t			flags1;
+#define MCX_CAP_DEVICE_END_PAD		0x80
+#define MCX_CAP_DEVICE_CC_QUERY_ALLOWED	0x40
+#define MCX_CAP_DEVICE_CC_MODIFY_ALLOWED \
+					0x20
+#define MCX_CAP_DEVICE_START_PAD	0x10
+#define MCX_CAP_DEVICE_128BYTE_CACHELINE \
+					0x08
+	uint8_t			__reserved__[1];
+	uint16_t		gid_table_size;
+
+	uint16_t		flags2;
+#define MCX_CAP_DEVICE_OUT_OF_SEQ_CNT	0x8000
+#define MCX_CAP_DEVICE_VPORT_COUNTERS	0x4000
+#define MCX_CAP_DEVICE_RETRANSMISSION_Q_COUNTERS \
+					0x2000
+#define MCX_CAP_DEVICE_DEBUG		0x1000
+#define MCX_CAP_DEVICE_MODIFY_RQ_COUNTERS_SET_ID \
+					0x8000
+#define MCX_CAP_DEVICE_RQ_DELAY_DROP	0x4000
+#define MCX_CAP_DEVICe_MAX_QP_CNT_MASK	0x03ff
+	uint16_t		pkey_table_size;
+
+	uint8_t			flags3;
+#define MCX_CAP_DEVICE_VPORT_GROUP_MANAGER \
+					0x80
+#define MCX_CAP_DEVICE_VHCA_GROUP_MANAGER \
+					0x40
+#define MCX_CAP_DEVICE_IB_VIRTUAL	0x20
+#define MCX_CAP_DEVICE_ETH_VIRTUAL	0x10
+#define MCX_CAP_DEVICE_ETS		0x04
+#define MCX_CAP_DEVICE_NIC_FLOW_TABLE	0x02
+#define MCX_CAP_DEVICE_ESWITCH_FLOW_TABLE \
+					0x01
+	uint8_t			local_ca_ack_delay; /* 5 bits */
+#define MCX_CAP_DEVICE_LOCAL_CA_ACK_DELAY \
+					0x1f
+	uint8_t			port_type;
+#define MCX_CAP_DEVICE_PORT_MODULE_EVENT \
+					0x80
+#define MCX_CAP_DEVICE_PORT_TYPE	0x03
+	uint8_t			num_ports;
+
+	uint8_t			snapshot_log_max_msg;
+#define MCX_CAP_DEVICE_SNAPSHOT		0x80
+#define MCX_CAP_DEVICE_LOG_MAX_MSG	0x1f
+	uint8_t			max_tc; /* 4 bits */
+#define MCX_CAP_DEVICE_MAX_TC		0x0f
+	uint8_t			flags4;
+#define MCX_CAP_DEVICE_TEMP_WARN_EVENT	0x80
+#define MCX_CAP_DEVICE_DCBX		0x40
+#define MCX_CAP_DEVICE_ROL_S		0x02
+#define MCX_CAP_DEVICE_ROL_G		0x01
+	uint8_t			wol;
+#define MCX_CAP_DEVICE_WOL_S		0x40
+#define MCX_CAP_DEVICE_WOL_G		0x20
+#define MCX_CAP_DEVICE_WOL_A		0x10
+#define MCX_CAP_DEVICE_WOL_B		0x08
+#define MCX_CAP_DEVICE_WOL_M		0x04
+#define MCX_CAP_DEVICE_WOL_U		0x02
+#define MCX_CAP_DEVICE_WOL_P		0x01
+
+	uint16_t		stat_rate_support;
+	uint8_t			__reserved__[1];
+	uint8_t			cqe_version; /* 4 bits */
+#define MCX_CAP_DEVICE_CQE_VERSION	0x0f
+
+	uint32_t		flags5;
+#define MCX_CAP_DEVICE_COMPACT_ADDRESS_VECTOR \
+					0x80000000
+#define MCX_CAP_DEVICE_STRIDING_RQ	0x40000000
+#define MCX_CAP_DEVICE_IPOIP_ENHANCED_OFFLOADS \
+					0x10000000
+#define MCX_CAP_DEVICE_IPOIP_IPOIP_OFFLOADS \
+					0x08000000
+#define MCX_CAP_DEVICE_DC_CONNECT_CP	0x00040000
+#define MCX_CAP_DEVICE_DC_CNAK_DRACE	0x00020000
+#define MCX_CAP_DEVICE_DRAIN_SIGERR	0x00010000
+#define MCX_CAP_DEVICE_DRAIN_SIGERR	0x00010000
+#define MCX_CAP_DEVICE_CMDIF_CHECKSUM	0x0000c000
+#define MCX_CAP_DEVICE_SIGERR_QCE	0x00002000
+#define MCX_CAP_DEVICE_WQ_SIGNATURE	0x00000800
+#define MCX_CAP_DEVICE_SCTR_DATA_CQE	0x00000400
+#define MCX_CAP_DEVICE_SHO		0x00000100
+#define MCX_CAP_DEVICE_TPH		0x00000080
+#define MCX_CAP_DEVICE_RF		0x00000040
+#define MCX_CAP_DEVICE_DCT		0x00000020
+#define MCX_CAP_DEVICE_QOS		0x00000010
+#define MCX_CAP_DEVICe_ETH_NET_OFFLOADS	0x00000008
+#define MCX_CAP_DEVICE_ROCE		0x00000004
+#define MCX_CAP_DEVICE_ATOMIC		0x00000002
+
+	uint32_t		flags6;
+#define MCX_CAP_DEVICE_CQ_OI		0x80000000
+#define MCX_CAP_DEVICE_CQ_RESIZE	0x40000000
+#define MCX_CAP_DEVICE_CQ_MODERATION	0x20000000
+#define MCX_CAP_DEVICE_CQ_PERIOD_MODE_MODIFY \
+					0x10000000
+#define MCX_CAP_DEVICE_CQ_INVALIDATE	0x08000000
+#define MCX_CAP_DEVICE_RESERVED_AT_255	0x04000000
+#define MCX_CAP_DEVICE_CQ_EQ_REMAP	0x02000000
+#define MCX_CAP_DEVICE_PG		0x01000000
+#define MCX_CAP_DEVICE_BLOCK_LB_MC	0x00800000
+#define MCX_CAP_DEVICE_EXPONENTIAL_BACKOFF \
+					0x00400000
+#define MCX_CAP_DEVICE_SCQE_BREAK_MODERATION \
+					0x00200000
+#define MCX_CAP_DEVICE_CQ_PERIOD_START_FROM_CQE \
+					0x00100000
+#define MCX_CAP_DEVICE_CD		0x00080000
+#define MCX_CAP_DEVICE_ATM		0x00040000
+#define MCX_CAP_DEVICE_APM		0x00020000
+#define MCX_CAP_DEVICE_IMAICL		0x00010000
+#define MCX_CAP_DEVICE_QKV		0x00000200
+#define MCX_CAP_DEVICE_PKV		0x00000100
+#define MCX_CAP_DEVICE_SET_DETH_SQPN	0x00000080
+#define MCX_CAP_DEVICE_XRC		0x00000008
+#define MCX_CAP_DEVICE_UD		0x00000004
+#define MCX_CAP_DEVICE_UC		0x00000002
+#define MCX_CAP_DEVICE_RC		0x00000001
+
+	uint8_t			uar_flags;
+#define MCX_CAP_DEVICE_UAR_4K		0x80
+	uint8_t			uar_sz;	/* 6 bits */
+#define MCX_CAP_DEVICE_UAR_SZ		0x3f
+	uint8_t			__reserved__[1];
+	uint8_t			log_pg_sz;
+
+	uint8_t			flags7;
+#define MCX_CAP_DEVICE_BF		0x80
+#define MCX_CAP_DEVICE_DRIVER_VERSION	0x40
+#define MCX_CAP_DEVICE_PAD_TX_ETH_PACKET \
+					0x20
+	uint8_t			log_bf_reg_size; /* 5 bits */
+#define MCX_CAP_DEVICE_LOG_BF_REG_SIZE	0x1f
+	uint8_t			__reserved__[2];
+
+	uint16_t		num_of_diagnostic_counters;
+	uint16_t		max_wqe_sz_sq;
+
+	uint8_t			__reserved__[2];
+	uint16_t		max_wqe_sz_rq;
+
+	uint8_t			__reserved__[2];
+	uint16_t		max_wqe_sz_sq_dc;
+
+	uint32_t		max_qp_mcg; /* 25 bits */
+#define MCX_CAP_DEVICE_MAX_QP_MCG	0x1ffffff
+
+	uint8_t			__reserved__[3];
+	uint8_t			log_max_mcq;
+
+	uint8_t			log_max_transport_domain; /* 5 bits */
+#define MCX_CAP_DEVICE_LOG_MAX_TRANSORT_DOMAIN \
+					0x1f
+	uint8_t			log_max_pd; /* 5 bits */
+#define MCX_CAP_DEVICE_LOG_MAX_PD	0x1f
+	uint8_t			__reserved__[1];
+	uint8_t			log_max_xrcd; /* 5 bits */
+#define MCX_CAP_DEVICE_LOG_MAX_XRCD	0x1f
+
+	uint8_t			__reserved__[2];
+	uint16_t		max_flow_counter;
+
+	uint8_t			log_max_rq; /* 5 bits */
+#define MCX_CAP_DEVICE_LOG_MAX_RQ	0x1f
+	uint8_t			log_max_sq; /* 5 bits */
+#define MCX_CAP_DEVICE_LOG_MAX_SQ	0x1f
+	uint8_t			log_max_tir; /* 5 bits */
+#define MCX_CAP_DEVICE_LOG_MAX_TIR	0x1f
+	uint8_t			log_max_tis; /* 5 bits */
+#define MCX_CAP_DEVICE_LOG_MAX_TIS	0x1f
+
+	uint8_t 		flags8;
+#define MCX_CAP_DEVICE_BASIC_CYCLIC_RCV_WQE \
+					0x80
+#define MCX_CAP_DEVICE_LOG_MAX_RMP	0x1f
+	uint8_t			log_max_rqt; /* 5 bits */
+#define MCX_CAP_DEVICE_LOG_MAX_RQT	0x1f
+	uint8_t			log_max_rqt_size; /* 5 bits */
+#define MCX_CAP_DEVICE_LOG_MAX_RQT_SIZE	0x1f
+	uint8_t			log_max_tis_per_sq; /* 5 bits */
+#define MCX_CAP_DEVICE_LOG_MAX_TIS_PER_SQ \
+					0x1f
+} __packed __aligned(8);
+
+CTASSERT(offsetof(struct mcx_cap_device, max_indirection) == 0x20);
+CTASSERT(offsetof(struct mcx_cap_device, flags1) == 0x2c);
+CTASSERT(offsetof(struct mcx_cap_device, flags2) == 0x30);
+CTASSERT(offsetof(struct mcx_cap_device, snapshot_log_max_msg) == 0x38);
+CTASSERT(offsetof(struct mcx_cap_device, flags5) == 0x40);
+CTASSERT(offsetof(struct mcx_cap_device, flags7) == 0x4c);
+CTASSERT(sizeof(struct mcx_cap_device) <= MCX_CMDQ_MAILBOX_DATASIZE);
+
+struct mcx_cmd_set_driver_version_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_set_driver_version_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_set_driver_version {
+	uint8_t			cmd_driver_version[64];
+} __packed __aligned(8);
+
+struct mcx_cmd_modify_nic_vport_context_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[4];
+	uint32_t		cmd_field_select;
+#define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_ADDR	0x04
+#define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC	0x10
+#define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU	0x40
+} __packed __aligned(4);
+
+struct mcx_cmd_modify_nic_vport_context_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_nic_vport_context_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[4];
+	uint8_t			cmd_allowed_list_type;
+	uint8_t			cmd_reserved2[3];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_nic_vport_context_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_nic_vport_ctx {
+	uint32_t		vp_min_wqe_inline_mode;
+	uint8_t			vp_reserved0[32];
+	uint32_t		vp_mtu;
+	uint8_t			vp_reserved1[200];
+	uint16_t		vp_flags;
+#define MCX_NIC_VPORT_CTX_LIST_UC_MAC			(0)
+#define MCX_NIC_VPORT_CTX_LIST_MC_MAC			(1 << 24)
+#define MCX_NIC_VPORT_CTX_LIST_VLAN			(2 << 24)
+#define MCX_NIC_VPORT_CTX_PROMISC_ALL			(1 << 13)
+#define MCX_NIC_VPORT_CTX_PROMISC_MCAST			(1 << 14)
+#define MCX_NIC_VPORT_CTX_PROMISC_UCAST			(1 << 15)
+	uint16_t		vp_allowed_list_size;
+	uint64_t		vp_perm_addr;
+	uint8_t			vp_reserved2[4];
+	/* allowed list follows */
+} __packed __aligned(4);
+
+struct mcx_counter {
+	uint64_t		packets;
+	uint64_t		octets;
+} __packed __aligned(4);
+
+struct mcx_nic_vport_counters {
+	struct mcx_counter	rx_err;
+	struct mcx_counter	tx_err;
+	uint8_t			reserved0[64]; /* 0x30 */
+	struct mcx_counter	rx_bcast;
+	struct mcx_counter	tx_bcast;
+	struct mcx_counter	rx_ucast;
+	struct mcx_counter	tx_ucast;
+	struct mcx_counter	rx_mcast;
+	struct mcx_counter	tx_mcast;
+	uint8_t			reserved1[0x210 - 0xd0];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_vport_counters_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_vport_counters_mb_in {
+	uint8_t			cmd_reserved0[8];
+	uint8_t			cmd_clear;
+	uint8_t			cmd_reserved1[7];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_vport_counters_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_flow_counter_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_flow_counter_mb_in {
+	uint8_t			cmd_reserved0[8];
+	uint8_t			cmd_clear;
+	uint8_t			cmd_reserved1[5];
+	uint16_t		cmd_flow_counter_id;
+} __packed __aligned(4);
+
+struct mcx_cmd_query_flow_counter_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_alloc_uar_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_alloc_uar_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint32_t		cmd_uar;
+	uint8_t			cmd_reserved1[4];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_special_ctx_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_special_ctx_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint8_t			cmd_reserved1[4];
+	uint32_t		cmd_resd_lkey;
+} __packed __aligned(4);
+
+struct mcx_eq_ctx {
+	uint32_t		eq_status;
+#define MCX_EQ_CTX_ST_SHIFT		8
+#define MCX_EQ_CTX_ST_MASK		(0xf << MCX_EQ_CTX_ST_SHIFT)
+#define MCX_EQ_CTX_ST_ARMED		(0x9 << MCX_EQ_CTX_ST_SHIFT)
+#define MCX_EQ_CTX_ST_FIRED		(0xa << MCX_EQ_CTX_ST_SHIFT)
+#define MCX_EQ_CTX_OI_SHIFT		17
+#define MCX_EQ_CTX_OI			(1 << MCX_EQ_CTX_OI_SHIFT)
+#define MCX_EQ_CTX_EC_SHIFT		18
+#define MCX_EQ_CTX_EC			(1 << MCX_EQ_CTX_EC_SHIFT)
+#define MCX_EQ_CTX_STATUS_SHIFT		28
+#define MCX_EQ_CTX_STATUS_MASK		(0xf << MCX_EQ_CTX_STATUS_SHIFT)
+#define MCX_EQ_CTX_STATUS_OK		(0x0 << MCX_EQ_CTX_STATUS_SHIFT)
+#define MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE \
+					(0xa << MCX_EQ_CTX_STATUS_SHIFT)
+	uint32_t		eq_reserved1;
+	uint32_t		eq_page_offset;
+#define MCX_EQ_CTX_PAGE_OFFSET_SHIFT	5
+	uint32_t		eq_uar_size;
+#define MCX_EQ_CTX_UAR_PAGE_MASK	0xffffff
+#define MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT	24
+	uint32_t		eq_reserved2;
+	uint8_t			eq_reserved3[3];
+	uint8_t			eq_intr;
+	uint32_t		eq_log_page_size;
+#define MCX_EQ_CTX_LOG_PAGE_SIZE_SHIFT	24
+	uint32_t		eq_reserved4[3];
+	uint32_t		eq_consumer_counter;
+	uint32_t		eq_producer_counter;
+#define MCX_EQ_CTX_COUNTER_MASK		0xffffff
+	uint32_t		eq_reserved5[4];
+} __packed __aligned(4);
+
+CTASSERT(sizeof(struct mcx_eq_ctx) == 64);
+
+struct mcx_cmd_create_eq_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_create_eq_mb_in {
+	struct mcx_eq_ctx	cmd_eq_ctx;
+	uint8_t			cmd_reserved0[8];
+	uint64_t		cmd_event_bitmask;
+#define MCX_EVENT_TYPE_COMPLETION	0x00
+#define MCX_EVENT_TYPE_CQ_ERROR		0x04
+#define MCX_EVENT_TYPE_INTERNAL_ERROR	0x08
+#define MCX_EVENT_TYPE_PORT_CHANGE	0x09
+#define MCX_EVENT_TYPE_CMD_COMPLETION	0x0a
+#define MCX_EVENT_TYPE_PAGE_REQUEST	0x0b
+#define MCX_EVENT_TYPE_LAST_WQE		0x13
+	uint8_t			cmd_reserved1[176];
+} __packed __aligned(4);
+
+struct mcx_cmd_create_eq_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint32_t		cmd_eqn;
+	uint8_t			cmd_reserved1[4];
+} __packed __aligned(4);
+
+struct mcx_eq_entry {
+	uint8_t			eq_reserved1;
+	uint8_t			eq_event_type;
+	uint8_t			eq_reserved2;
+	uint8_t			eq_event_sub_type;
+
+	uint8_t			eq_reserved3[28];
+	uint32_t		eq_event_data[7];
+	uint8_t			eq_reserved4[2];
+	uint8_t			eq_signature;
+	uint8_t			eq_owner;
+#define MCX_EQ_ENTRY_OWNER_INIT			1
+} __packed __aligned(4);
+
+CTASSERT(sizeof(struct mcx_eq_entry) == 64);
+
+struct mcx_cmd_alloc_pd_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_alloc_pd_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint32_t		cmd_pd;
+	uint8_t			cmd_reserved1[4];
+} __packed __aligned(4);
+
+struct mcx_cmd_alloc_td_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_alloc_td_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint32_t		cmd_tdomain;
+	uint8_t			cmd_reserved1[4];
+} __packed __aligned(4);
+
+struct mcx_cmd_create_tir_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_create_tir_mb_in {
+	uint8_t			cmd_reserved0[20];
+	uint32_t		cmd_disp_type;
+#define MCX_TIR_CTX_DISP_TYPE_SHIFT	28
+	uint8_t			cmd_reserved1[8];
+	uint32_t		cmd_lro;
+	uint8_t			cmd_reserved2[8];
+	uint32_t		cmd_inline_rqn;
+	uint32_t		cmd_indir_table;
+	uint32_t		cmd_tdomain;
+	uint8_t			cmd_rx_hash_key[40];
+	uint32_t		cmd_rx_hash_sel_outer;
+	uint32_t		cmd_rx_hash_sel_inner;
+	uint8_t			cmd_reserved3[152];
+} __packed __aligned(4);
+
+struct mcx_cmd_create_tir_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint32_t		cmd_tirn;
+	uint8_t			cmd_reserved1[4];
+} __packed __aligned(4);
+
+struct mcx_cmd_create_tis_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_create_tis_mb_in {
+	uint8_t			cmd_reserved[16];
+	uint32_t		cmd_prio;
+	uint8_t			cmd_reserved1[32];
+	uint32_t		cmd_tdomain;
+	uint8_t			cmd_reserved2[120];
+} __packed __aligned(4);
+
+struct mcx_cmd_create_tis_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint32_t		cmd_tisn;
+	uint8_t			cmd_reserved1[4];
+} __packed __aligned(4);
+
+struct mcx_cq_ctx {
+	uint32_t		cq_status;
+	uint32_t		cq_reserved1;
+	uint32_t		cq_page_offset;
+	uint32_t		cq_uar_size;
+#define MCX_CQ_CTX_UAR_PAGE_MASK	0xffffff
+#define MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT	24
+	uint32_t		cq_period_max_count;
+#define MCX_CQ_CTX_PERIOD_SHIFT		16
+	uint32_t		cq_eqn;
+	uint32_t		cq_log_page_size;
+#define MCX_CQ_CTX_LOG_PAGE_SIZE_SHIFT	24
+	uint32_t		cq_reserved2;
+	uint32_t		cq_last_notified;
+	uint32_t		cq_last_solicit;
+	uint32_t		cq_consumer_counter;
+	uint32_t		cq_producer_counter;
+	uint8_t			cq_reserved3[8];
+	uint64_t		cq_doorbell;
+} __packed __aligned(4);
+
+CTASSERT(sizeof(struct mcx_cq_ctx) == 64);
+
+struct mcx_cmd_create_cq_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_create_cq_mb_in {
+	struct mcx_cq_ctx	cmd_cq_ctx;
+	uint8_t			cmd_reserved1[192];
+} __packed __aligned(4);
+
+struct mcx_cmd_create_cq_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint32_t		cmd_cqn;
+	uint8_t			cmd_reserved1[4];
+} __packed __aligned(4);
+
+struct mcx_cq_entry {
+	uint32_t		cq_reserved1;
+	uint32_t		cq_lro;
+	uint32_t		cq_lro_ack_seq_num;
+	uint32_t		cq_rx_hash;
+	uint32_t		cq_rx_hash_type;
+	uint32_t		cq_checksum;
+	uint32_t		cq_reserved2;
+	uint32_t		cq_flags;
+	uint32_t		cq_lro_srqn;
+	uint32_t		cq_reserved3[2];
+	uint32_t		cq_byte_cnt;
+	uint32_t		cq_lro_ts_value;
+	uint32_t		cq_lro_ts_echo;
+	uint32_t		cq_flow_tag;
+	uint16_t		cq_wqe_count;
+	uint8_t			cq_signature;
+	uint8_t			cq_opcode_owner;
+#define MCX_CQ_ENTRY_FLAG_OWNER			(1 << 0)
+#define MCX_CQ_ENTRY_FLAG_SE			(1 << 1)
+#define MCX_CQ_ENTRY_FORMAT_SHIFT		2
+#define MCX_CQ_ENTRY_OPCODE_SHIFT		4
+
+#define MCX_CQ_ENTRY_FORMAT_NO_INLINE		0
+#define MCX_CQ_ENTRY_FORMAT_INLINE_32		1
+#define MCX_CQ_ENTRY_FORMAT_INLINE_64		2
+#define MCX_CQ_ENTRY_FORMAT_COMPRESSED		3
+
+#define MCX_CQ_ENTRY_OPCODE_REQ			0
+#define MCX_CQ_ENTRY_OPCODE_SEND		2
+#define MCX_CQ_ENTRY_OPCODE_REQ_ERR		13
+#define MCX_CQ_ENTRY_OPCODE_SEND_ERR		14
+#define MCX_CQ_ENTRY_OPCODE_INVALID		15
+
+} __packed __aligned(4);
+
+CTASSERT(sizeof(struct mcx_cq_entry) == 64);
+
+struct mcx_cq_doorbell {
+	uint32_t		 db_update_ci;
+	uint32_t		 db_arm_ci;
+#define MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT	28
+#define MCX_CQ_DOORBELL_ARM_CMD			(1 << 24)
+#define MCX_CQ_DOORBELL_ARM_CI_MASK		(0xffffff)
+} __packed __aligned(8);
+
+struct mcx_wq_ctx {
+	uint8_t			 wq_type;
+#define MCX_WQ_CTX_TYPE_CYCLIC			(1 << 4)
+#define MCX_WQ_CTX_TYPE_SIGNATURE		(1 << 3)
+	uint8_t			 wq_reserved0[5];
+	uint16_t		 wq_lwm;
+	uint32_t		 wq_pd;
+	uint32_t		 wq_uar_page;
+	uint64_t		 wq_doorbell;
+	uint32_t		 wq_hw_counter;
+	uint32_t		 wq_sw_counter;
+	uint16_t		 wq_log_stride;
+	uint8_t			 wq_log_page_sz;
+	uint8_t			 wq_log_size;
+	uint8_t			 wq_reserved1[156];
+} __packed __aligned(4);
+
+CTASSERT(sizeof(struct mcx_wq_ctx) == 0xC0);
+
+struct mcx_sq_ctx {
+	uint32_t		sq_flags;
+#define MCX_SQ_CTX_RLKEY			(1 << 31)
+#define MCX_SQ_CTX_FRE_SHIFT			(1 << 29)
+#define MCX_SQ_CTX_FLUSH_IN_ERROR		(1 << 28)
+#define MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT		24
+#define MCX_SQ_CTX_STATE_SHIFT			20
+	uint32_t		sq_user_index;
+	uint32_t		sq_cqn;
+	uint32_t		sq_reserved1[5];
+	uint32_t		sq_tis_lst_sz;
+#define MCX_SQ_CTX_TIS_LST_SZ_SHIFT		16
+	uint32_t		sq_reserved2[2];
+	uint32_t		sq_tis_num;
+	struct mcx_wq_ctx	sq_wq;
+} __packed __aligned(4);
+
+struct mcx_sq_entry {
+	/* control segment */
+	uint32_t		sqe_opcode_index;
+#define MCX_SQE_WQE_INDEX_SHIFT			8
+#define MCX_SQE_WQE_OPCODE_NOP			0x00
+#define MCX_SQE_WQE_OPCODE_SEND			0x0a
+	uint32_t		sqe_ds_sq_num;
+#define MCX_SQE_SQ_NUM_SHIFT			8
+	uint32_t		sqe_signature;
+#define MCX_SQE_SIGNATURE_SHIFT			24
+#define MCX_SQE_SOLICITED_EVENT			0x02
+#define MCX_SQE_CE_CQE_ON_ERR			0x00
+#define MCX_SQE_CE_CQE_FIRST_ERR		0x04
+#define MCX_SQE_CE_CQE_ALWAYS			0x08
+#define MCX_SQE_CE_CQE_SOLICIT			0x0C
+#define MCX_SQE_FM_NO_FENCE			0x00
+#define MCX_SQE_FM_SMALL_FENCE			0x40
+	uint32_t		sqe_mkey;
+
+	/* ethernet segment */
+	uint32_t		sqe_reserved1;
+	uint32_t		sqe_mss_csum;
+#define MCX_SQE_L4_CSUM				(1 << 31)
+#define MCX_SQE_L3_CSUM				(1 << 30)
+	uint32_t		sqe_reserved2;
+	uint16_t		sqe_inline_header_size;
+	uint16_t		sqe_inline_headers[9];
+
+	/* data segment */
+	uint32_t		sqe_byte_count;
+	uint32_t		sqe_lkey;
+	uint64_t		sqe_addr;
+
+	/* possibly more data segments? */
+} __packed __aligned(64);
+
+CTASSERT(sizeof(struct mcx_sq_entry) == 64);
+
+struct mcx_cmd_create_sq_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_create_sq_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint32_t		cmd_sqn;
+	uint8_t			cmd_reserved1[4];
+} __packed __aligned(4);
+
+struct mcx_cmd_modify_sq_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint32_t		cmd_sq_state;
+	uint8_t			cmd_reserved1[4];
+} __packed __aligned(4);
+
+struct mcx_cmd_modify_sq_mb_in {
+	uint32_t		cmd_modify_hi;
+	uint32_t		cmd_modify_lo;
+	uint8_t			cmd_reserved0[8];
+	struct mcx_sq_ctx	cmd_sq_ctx;
+} __packed __aligned(4);
+
+struct mcx_cmd_modify_sq_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_rq_ctx {
+	uint32_t		rq_flags;
+#define MCX_RQ_CTX_RLKEY			(1 << 31)
+#define MCX_RQ_CTX_VLAN_STRIP_DIS		(1 << 28)
+#define MCX_RQ_CTX_MEM_RQ_TYPE_SHIFT		24
+#define MCX_RQ_CTX_STATE_SHIFT			20
+#define MCX_RQ_CTX_FLUSH_IN_ERROR		(1 << 18)
+	uint32_t		rq_user_index;
+	uint32_t		rq_cqn;
+	uint32_t		rq_reserved1;
+	uint32_t		rq_rmpn;
+	uint32_t		rq_reserved2[7];
+	struct mcx_wq_ctx	rq_wq;
+} __packed __aligned(4);
+
+struct mcx_rq_entry {
+	uint32_t		rqe_byte_count;
+	uint32_t		rqe_lkey;
+	uint64_t		rqe_addr;
+} __packed __aligned(16);
+
+struct mcx_cmd_create_rq_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_create_rq_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint32_t		cmd_rqn;
+	uint8_t			cmd_reserved1[4];
+} __packed __aligned(4);
+
+struct mcx_cmd_modify_rq_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint32_t		cmd_rq_state;
+	uint8_t			cmd_reserved1[4];
+} __packed __aligned(4);
+
+struct mcx_cmd_modify_rq_mb_in {
+	uint32_t		cmd_modify_hi;
+	uint32_t		cmd_modify_lo;
+	uint8_t			cmd_reserved0[8];
+	struct mcx_rq_ctx	cmd_rq_ctx;
+} __packed __aligned(4);
+
+struct mcx_cmd_modify_rq_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_create_flow_table_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_flow_table_ctx {
+	uint8_t			ft_miss_action;
+	uint8_t			ft_level;
+	uint8_t			ft_reserved0;
+	uint8_t			ft_log_size;
+	uint32_t		ft_table_miss_id;
+	uint8_t			ft_reserved1[28];
+} __packed __aligned(4);
+
+struct mcx_cmd_create_flow_table_mb_in {
+	uint8_t			cmd_table_type;
+	uint8_t			cmd_reserved0[7];
+	struct mcx_flow_table_ctx cmd_ctx;
+} __packed __aligned(4);
+
+struct mcx_cmd_create_flow_table_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint32_t		cmd_table_id;
+	uint8_t			cmd_reserved1[4];
+} __packed __aligned(4);
+
+struct mcx_cmd_set_flow_table_root_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_set_flow_table_root_mb_in {
+	uint8_t			cmd_table_type;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_table_id;
+	uint8_t			cmd_reserved1[56];
+} __packed __aligned(4);
+
+struct mcx_cmd_set_flow_table_root_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_create_flow_group_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_create_flow_group_mb_in {
+	uint8_t			cmd_table_type;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_table_id;
+	uint8_t			cmd_reserved1[4];
+	uint32_t		cmd_start_flow_index;
+	uint8_t			cmd_reserved2[4];
+	uint32_t		cmd_end_flow_index;
+	uint8_t			cmd_reserved3[23];
+	uint8_t			cmd_match_criteria_enable;
+#define MCX_CREATE_FLOW_GROUP_CRIT_OUTER	(1 << 0)
+#define MCX_CREATE_FLOW_GROUP_CRIT_MISC		(1 << 1)
+#define MCX_CREATE_FLOW_GROUP_CRIT_INNER	(1 << 2)
+	uint8_t			cmd_match_criteria[512];
+	uint8_t			cmd_reserved4[448];
+} __packed __aligned(4);
+
+struct mcx_cmd_create_flow_group_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint32_t		cmd_group_id;
+	uint8_t			cmd_reserved1[4];
+} __packed __aligned(4);
+
+struct mcx_flow_ctx {
+	uint8_t			fc_reserved0[4];
+	uint32_t		fc_group_id;
+	uint32_t		fc_flow_tag;
+	uint32_t		fc_action;
+#define MCX_FLOW_CONTEXT_ACTION_ALLOW		(1 << 0)
+#define MCX_FLOW_CONTEXT_ACTION_DROP		(1 << 1)
+#define MCX_FLOW_CONTEXT_ACTION_FORWARD		(1 << 2)
+#define MCX_FLOW_CONTEXT_ACTION_COUNT		(1 << 3)
+	uint32_t		fc_dest_list_size;
+	uint32_t		fc_counter_list_size;
+	uint8_t			fc_reserved1[40];
+	uint8_t			fc_match_value[512];
+	uint8_t			fc_reserved2[192];
+} __packed __aligned(4);
+
+#define MCX_FLOW_CONTEXT_DEST_TYPE_TABLE	(1 << 24)
+#define MCX_FLOW_CONTEXT_DEST_TYPE_TIR		(2 << 24)
+
+struct mcx_cmd_set_flow_table_entry_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_set_flow_table_entry_mb_in {
+	uint8_t			cmd_table_type;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_table_id;
+	uint32_t		cmd_modify_enable_mask;
+	uint8_t			cmd_reserved1[4];
+	uint32_t		cmd_flow_index;
+	uint8_t			cmd_reserved2[28];
+	struct mcx_flow_ctx	cmd_flow_ctx;
+} __packed __aligned(4);
+
+struct mcx_cmd_set_flow_table_entry_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_flow_table_entry_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_flow_table_entry_mb_in {
+	uint8_t			cmd_table_type;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_table_id;
+	uint8_t			cmd_reserved1[8];
+	uint32_t		cmd_flow_index;
+	uint8_t			cmd_reserved2[28];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_flow_table_entry_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_flow_table_entry_mb_out {
+	uint8_t			cmd_reserved0[48];
+	struct mcx_flow_ctx	cmd_flow_ctx;
+} __packed __aligned(4);
+
+struct mcx_cmd_query_flow_group_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_flow_group_mb_in {
+	uint8_t			cmd_table_type;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_table_id;
+	uint32_t		cmd_group_id;
+	uint8_t			cmd_reserved1[36];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_flow_group_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_flow_group_mb_out {
+	uint8_t			cmd_reserved0[12];
+	uint32_t		cmd_start_flow_index;
+	uint8_t			cmd_reserved1[4];
+	uint32_t		cmd_end_flow_index;
+	uint8_t			cmd_reserved2[20];
+	uint32_t		cmd_match_criteria_enable;
+	uint8_t			cmd_match_criteria[512];
+	uint8_t			cmd_reserved4[448];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_flow_table_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_flow_table_mb_in {
+	uint8_t			cmd_table_type;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_table_id;
+	uint8_t			cmd_reserved1[40];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_flow_table_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_flow_table_mb_out {
+	uint8_t			cmd_reserved0[4];
+	struct mcx_flow_table_ctx cmd_ctx;
+} __packed __aligned(4);
+
+struct mcx_cmd_alloc_flow_counter_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_rq_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint32_t		cmd_rqn;
+	uint8_t			cmd_reserved1[4];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_rq_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_rq_mb_out {
+	uint8_t			cmd_reserved0[16];
+	struct mcx_rq_ctx	cmd_ctx;
+};
+
+struct mcx_cmd_query_sq_in {
+	uint16_t		cmd_opcode;
+	uint8_t			cmd_reserved0[4];
+	uint16_t		cmd_op_mod;
+	uint32_t		cmd_sqn;
+	uint8_t			cmd_reserved1[4];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_sq_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint8_t			cmd_reserved1[8];
+} __packed __aligned(4);
+
+struct mcx_cmd_query_sq_mb_out {
+	uint8_t			cmd_reserved0[16];
+	struct mcx_sq_ctx	cmd_ctx;
+};
+
+struct mcx_cmd_alloc_flow_counter_out {
+	uint8_t			cmd_status;
+	uint8_t			cmd_reserved0[3];
+	uint32_t		cmd_syndrome;
+	uint8_t			cmd_reserved1[2];
+	uint16_t		cmd_flow_counter_id;
+	uint8_t			cmd_reserved2[4];
+} __packed __aligned(4);
+
+struct mcx_wq_doorbell {
+	uint32_t		 db_recv_counter;
+	uint32_t		 db_send_counter;
+} __packed __aligned(8);
+
+struct mcx_dmamem {
+	bus_dmamap_t		 mxm_map;
+	bus_dma_segment_t	 mxm_seg;
+	int			 mxm_nsegs;
+	size_t			 mxm_size;
+	caddr_t			 mxm_kva;
+};
+#define MCX_DMA_MAP(_mxm)	((_mxm)->mxm_map)
+#define MCX_DMA_DVA(_mxm)	((_mxm)->mxm_map->dm_segs[0].ds_addr)
+#define MCX_DMA_KVA(_mxm)	((void *)(_mxm)->mxm_kva)
+#define MCX_DMA_LEN(_mxm)	((_mxm)->mxm_size)
+
+struct mcx_hwmem {
+	bus_dmamap_t		 mhm_map;
+	bus_dma_segment_t	*mhm_segs;
+	unsigned int		 mhm_seg_count;
+	unsigned int		 mhm_npages;
+};
+
+struct mcx_slot {
+	bus_dmamap_t		 ms_map;
+	struct mbuf		*ms_m;
+};
+
+struct mcx_cq {
+	int			 cq_n;
+	struct mcx_dmamem	 cq_mem;
+	uint32_t		*cq_doorbell;
+	int			 cq_cons;
+	int			 cq_count;
+};
+
+struct mcx_softc {
+	struct device		 sc_dev;
+	struct arpcom		 sc_ac;
+	struct ifmedia		 sc_media;
+	uint64_t		 sc_media_status;
+	uint64_t		 sc_media_active;
+
+	pci_chipset_tag_t	 sc_pc;
+	pci_intr_handle_t	 sc_ih;
+	void			*sc_ihc;
+	pcitag_t		 sc_tag;
+
+	bus_dma_tag_t		 sc_dmat;
+	bus_space_tag_t		 sc_memt;
+	bus_space_handle_t	 sc_memh;
+	bus_size_t		 sc_mems;
+
+	struct mcx_dmamem	 sc_cmdq_mem;
+	unsigned int		 sc_cmdq_mask;
+	unsigned int		 sc_cmdq_size;
+
+	unsigned int		 sc_cmdq_token;
+
+	struct mcx_hwmem	 sc_boot_pages;
+	struct mcx_hwmem	 sc_init_pages;
+	struct mcx_hwmem	 sc_regular_pages;
+
+	int			 sc_uar;
+	int			 sc_pd;
+	int			 sc_tdomain;
+	uint32_t		 sc_lkey;
+
+	struct mcx_dmamem	 sc_doorbell_mem;
+
+	int			 sc_eqn;
+	int			 sc_eq_cons;
+	struct mcx_dmamem	 sc_eq_mem;
+	int			 sc_flow_table_id;
+	int			 sc_flow_group_id;
+	uint16_t		 sc_flow_counter_id[4];
+	int			 sc_flow_table_entry;
+
+	struct mcx_cq		 sc_cq[MCX_MAX_CQS];
+	int			 sc_num_cq;
+
+	/* rx */
+	int			 sc_tirn;
+	int			 sc_rqn;
+	struct mcx_dmamem	 sc_rq_mem;
+	struct mcx_slot		*sc_rx_slots;
+	uint32_t		*sc_rx_doorbell;
+
+	int			 sc_rx_cons;
+	int			 sc_rx_prod;
+	struct timeout		 sc_rx_refill;
+	struct if_rxring	 sc_rxr;
+
+	/* tx */
+	int			 sc_tisn;
+	int			 sc_sqn;
+	struct mcx_dmamem	 sc_sq_mem;
+	struct mcx_slot		*sc_tx_slots;
+	uint32_t		*sc_tx_doorbell;
+	int			 sc_bf_size;
+	int			 sc_bf_offset;
+
+	int			 sc_tx_cons;
+	int			 sc_tx_prod;
+
+	uint64_t		 sc_last_cq_db;
+	uint64_t		 sc_last_srq_db;
+};
+#define DEVNAME(_sc) ((_sc)->sc_dev.dv_xname)
+
+static int	mcx_match(struct device *, void *, void *);
+static void	mcx_attach(struct device *, struct device *, void *);
+
+static int	mcx_version(struct mcx_softc *);
+static int	mcx_init_wait(struct mcx_softc *);
+static int	mcx_enable_hca(struct mcx_softc *);
+static int	mcx_teardown_hca(struct mcx_softc *, uint16_t);
+static int	mcx_read_hca_reg(struct mcx_softc *, uint16_t, void *, int);
+#if 0
+static int	mcx_write_hca_reg(struct mcx_softc *, uint16_t, const void *, int);
+#endif
+static int	mcx_issi(struct mcx_softc *);
+static int	mcx_pages(struct mcx_softc *, struct mcx_hwmem *, uint16_t);
+static int	mcx_hca_max_caps(struct mcx_softc *);
+static int	mcx_hca_set_caps(struct mcx_softc *);
+static int	mcx_init_hca(struct mcx_softc *);
+static int	mcx_set_driver_version(struct mcx_softc *);
+static int	mcx_iff(struct mcx_softc *);
+static int	mcx_alloc_uar(struct mcx_softc *);
+static int	mcx_alloc_pd(struct mcx_softc *);
+static int	mcx_alloc_tdomain(struct mcx_softc *);
+static int	mcx_create_eq(struct mcx_softc *);
+static int	mcx_query_nic_vport_context(struct mcx_softc *);
+static int	mcx_query_special_contexts(struct mcx_softc *);
+static int	mcx_create_cq(struct mcx_softc *, int);
+static int	mcx_create_sq(struct mcx_softc *, int);
+static int	mcx_ready_sq(struct mcx_softc *);
+static int	mcx_create_rq(struct mcx_softc *, int);
+static int	mcx_ready_rq(struct mcx_softc *);
+static int	mcx_create_tir(struct mcx_softc *);
+static int	mcx_create_tis(struct mcx_softc *);
+static int	mcx_create_flow_table(struct mcx_softc *);
+static int	mcx_set_flow_table_root(struct mcx_softc *);
+static int	mcx_create_flow_group(struct mcx_softc *);
+static int	mcx_set_flow_table_entry(struct mcx_softc *, int);
+
+#if 0
+static int	mcx_dump_flow_table(struct mcx_softc *);
+static int	mcx_dump_flow_table_entry(struct mcx_softc *, int);
+static int	mcx_dump_flow_group(struct mcx_softc *);
+static int	mcx_dump_rq(struct mcx_softc *);
+static int	mcx_dump_sq(struct mcx_softc *);
+#endif
+
+
+/*
+static void	mcx_cmdq_dump(const struct mcx_cmdq_entry *);
+static void	mcx_cmdq_mbox_dump(struct mcx_dmamem *, int);
+*/
+static void	mcx_refill(void *);
+static void	mcx_process_rx(struct mcx_softc *, struct mcx_cq_entry *,
+		    struct mbuf_list *, int *);
+static void	mcx_process_txeof(struct mcx_softc *, struct mcx_cq_entry *,
+		    int *);
+static void	mcx_process_cq(struct mcx_softc *, struct mcx_cq *);
+
+static void	mcx_arm_cq(struct mcx_softc *, struct mcx_cq *);
+static void	mcx_arm_eq(struct mcx_softc *);
+static int	mcx_intr(void *);
+
+static void	mcx_up(struct mcx_softc *);
+static void	mcx_down(struct mcx_softc *);
+static int	mcx_ioctl(struct ifnet *, u_long, caddr_t);
+static int	mcx_rxrinfo(struct mcx_softc *, struct if_rxrinfo *);
+static void	mcx_start(struct ifqueue *);
+static void	mcx_watchdog(struct ifnet *);
+static void	mcx_media_status(struct ifnet *, struct ifmediareq *);
+static int	mcx_media_change(struct ifnet *);
+
+static inline uint32_t
+		mcx_rd(struct mcx_softc *, bus_size_t);
+static inline void
+		mcx_wr(struct mcx_softc *, bus_size_t, uint32_t);
+static inline void
+		mcx_bar(struct mcx_softc *, bus_size_t, bus_size_t, int);
+
+static int	mcx_dmamem_alloc(struct mcx_softc *, struct mcx_dmamem *,
+		    bus_size_t, u_int align);
+static void	mcx_dmamem_zero(struct mcx_dmamem *);
+static void	mcx_dmamem_free(struct mcx_softc *, struct mcx_dmamem *);
+
+static int	mcx_hwmem_alloc(struct mcx_softc *, struct mcx_hwmem *,
+		    unsigned int);
+static void	mcx_hwmem_free(struct mcx_softc *, struct mcx_hwmem *);
+
+struct cfdriver mcx_cd = {
+	NULL,
+	"mcx",
+	DV_IFNET,
+};
+
+struct cfattach mcx_ca = {
+	sizeof(struct mcx_softc),
+	mcx_match,
+	mcx_attach,
+};
+
+static const struct pci_matchid mcx_devices[] = {
+	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27700 },
+	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27710 },
+};
+
+static int
+mcx_match(struct device *parent, void *match, void *aux)
+{
+	return (pci_matchbyid(aux, mcx_devices, nitems(mcx_devices)));
+}
+
+void
+mcx_attach(struct device *parent, struct device *self, void *aux)
+{
+	struct mcx_softc *sc = (struct mcx_softc *)self;
+	struct ifnet *ifp = &sc->sc_ac.ac_if;
+	struct pci_attach_args *pa = aux;
+	pcireg_t memtype;
+	uint32_t r;
+	unsigned int cq_stride;
+	unsigned int cq_size;
+	const char *intrstr;
+
+	sc->sc_pc = pa->pa_pc;
+	sc->sc_tag = pa->pa_tag;
+	sc->sc_dmat = pa->pa_dmat;
+
+	/* Map the PCI memory space */
+	memtype = pci_mapreg_type(sc->sc_pc, sc->sc_tag, MCX_HCA_BAR);
+	if (pci_mapreg_map(pa, MCX_HCA_BAR, memtype,
+	    BUS_SPACE_MAP_PREFETCHABLE, &sc->sc_memt, &sc->sc_memh,
+	    NULL, &sc->sc_mems, 0)) {
+		printf(": unable to map register memory\n");
+		return;
+	}
+
+	if (mcx_version(sc) != 0) {
+		/* error printed by mcx_version */
+		goto unmap;
+	}
+
+	r = mcx_rd(sc, MCX_CMDQ_ADDR_LO);
+	cq_stride = 1 << MCX_CMDQ_LOG_STRIDE(r); /* size of the entries */
+	cq_size = 1 << MCX_CMDQ_LOG_SIZE(r); /* number of entries */
+	if (cq_size > MCX_MAX_CQE) {
+		printf(", command queue size overflow %u\n", cq_size);
+		goto unmap;
+	}
+	if (cq_stride < sizeof(struct mcx_cmdq_entry)) {
+		printf(", command queue entry size underflow %u\n", cq_stride);
+		goto unmap;
+	}
+	if (cq_stride * cq_size > MCX_PAGE_SIZE) {
+		printf(", command queue page overflow\n");
+		goto unmap;
+	}
+
+	if (mcx_dmamem_alloc(sc, &sc->sc_doorbell_mem, MCX_PAGE_SIZE,
+	    MCX_PAGE_SIZE) != 0) {
+		printf(", unable to allocate doorbell memory\n");
+		goto unmap;
+	}
+
+	if (mcx_dmamem_alloc(sc, &sc->sc_cmdq_mem, MCX_PAGE_SIZE,
+	    MCX_PAGE_SIZE) != 0) {
+		printf(", unable to allocate command queue\n");
+		goto dbfree;
+	}
+
+	mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32);
+	mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint32_t), BUS_SPACE_BARRIER_WRITE);
+	mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem));
+	mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint32_t), BUS_SPACE_BARRIER_WRITE);
+
+	if (mcx_init_wait(sc) != 0) {
+		printf(", timeout waiting for init\n");
+		goto cqfree;
+	}
+
+	sc->sc_cmdq_mask = cq_size - 1;
+	sc->sc_cmdq_size = cq_stride;
+
+	if (mcx_enable_hca(sc) != 0) {
+		/* error printed by mcx_enable_hca */
+		goto cqfree;
+	}
+
+	if (mcx_issi(sc) != 0) {
+		/* error printed by mcx_issi */
+		goto teardown;
+	}
+
+	if (mcx_pages(sc, &sc->sc_boot_pages,
+	    htobe16(MCX_CMD_QUERY_PAGES_BOOT)) != 0) {
+		/* error printed by mcx_pages */
+		goto teardown;
+	}
+
+	if (mcx_hca_max_caps(sc) != 0) {
+		/* error printed by mcx_hca_max_caps */
+		goto teardown;
+	}
+
+	if (mcx_hca_set_caps(sc) != 0) {
+		/* error printed by mcx_hca_set_caps */
+		goto teardown;
+	}
+
+	if (mcx_pages(sc, &sc->sc_init_pages,
+	    htobe16(MCX_CMD_QUERY_PAGES_INIT)) != 0) {
+		/* error printed by mcx_pages */
+		goto teardown;
+	}
+
+	if (mcx_init_hca(sc) != 0) {
+		/* error printed by mcx_init_hca */
+		goto teardown;
+	}
+
+	if (mcx_pages(sc, &sc->sc_regular_pages,
+	    htobe16(MCX_CMD_QUERY_PAGES_REGULAR)) != 0) {
+		/* error printed by mcx_pages */
+		goto teardown;
+	}
+
+	/* apparently not necessary? */
+	if (mcx_set_driver_version(sc) != 0) {
+		/* error printed by mcx_set_driver_version */
+		goto teardown;
+	}
+
+	if (mcx_iff(sc) != 0) {	/* modify nic vport context */
+		/* error printed by mcx_iff? */
+		goto teardown;
+	}
+
+	if (mcx_alloc_uar(sc) != 0) {
+		/* error printed by mcx_alloc_uar */
+		goto teardown;
+	}
+
+	if (mcx_alloc_pd(sc) != 0) {
+		/* error printed by mcx_alloc_pd */
+		goto teardown;
+	}
+
+	if (mcx_alloc_tdomain(sc) != 0) {
+		/* error printed by mcx_alloc_tdomain */
+		goto teardown;
+	}
+
+	if (mcx_create_eq(sc) != 0) {
+		/* error printed by mcx_create_eq */
+		goto teardown;
+	}
+
+	if (mcx_query_nic_vport_context(sc) != 0) {
+		/* error printed by mcx_query_nic_vport_context */
+		goto teardown;
+	}
+
+	if (mcx_query_special_contexts(sc) != 0) {
+		/* error printed by mcx_query_special_contexts */
+		goto teardown;
+	}
+
+	/* PRM makes no mention of msi interrupts, just legacy and msi-x */
+	if (pci_intr_map_msix(pa, 0, &sc->sc_ih) != 0 &&
+	    pci_intr_map(pa, &sc->sc_ih) != 0) {
+		printf(": unable to map interrupt\n");
+		goto teardown;
+	}
+	intrstr = pci_intr_string(sc->sc_pc, sc->sc_ih);
+	sc->sc_ihc = pci_intr_establish(sc->sc_pc, sc->sc_ih, IPL_NET | IPL_MPSAFE,
+	    mcx_intr, sc, DEVNAME(sc));
+	if (sc->sc_ihc == NULL) {
+		printf(": unable to establish interrupt");
+		if (intrstr != NULL)
+			printf(" at %s", intrstr);
+		printf("\n");
+		goto teardown;
+	}
+	printf(", %s, address %s\n", intrstr, ether_sprintf(sc->sc_ac.ac_enaddr));
+
+	strlcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ);
+	ifp->if_softc = sc;
+	ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX;
+	ifp->if_xflags = IFXF_MPSAFE;
+	ifp->if_ioctl = mcx_ioctl;
+	ifp->if_qstart = mcx_start;
+	ifp->if_watchdog = mcx_watchdog;
+	ifp->if_hardmtu = MCX_HARDMTU;
+	ifp->if_capabilities = IFCAP_VLAN_MTU;
+	IFQ_SET_MAXLEN(&ifp->if_snd, 1024);
+
+	ifmedia_init(&sc->sc_media, IFM_IMASK, mcx_media_change,
+	    mcx_media_status);
+
+	if_attach(ifp);
+	ether_ifattach(ifp);
+
+	mcx_media_status(ifp, NULL);
+
+	timeout_set(&sc->sc_rx_refill, mcx_refill, sc);
+
+	/* set interface admin down, so bringing it up will start autoneg */
+
+	return;
+
+teardown:
+	mcx_teardown_hca(sc, htobe16(MCX_CMD_TEARDOWN_HCA_GRACEFUL));
+	/* error printed by mcx_teardown_hca, and we're already unwinding */
+cqfree:
+	mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32);
+	mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t), BUS_SPACE_BARRIER_WRITE);
+	mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem) |
+	    MCX_CMDQ_INTERFACE_DISABLED);
+	mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint64_t), BUS_SPACE_BARRIER_WRITE);
+
+	mcx_wr(sc, MCX_CMDQ_ADDR_HI, 0);
+	mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t), BUS_SPACE_BARRIER_WRITE);
+	mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_CMDQ_INTERFACE_DISABLED);
+
+	mcx_dmamem_free(sc, &sc->sc_cmdq_mem);
+dbfree:
+	mcx_dmamem_free(sc, &sc->sc_doorbell_mem);
+unmap:
+	bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems);
+	sc->sc_mems = 0;
+}
+
+static int
+mcx_version(struct mcx_softc *sc)
+{
+	uint32_t fw0, fw1;
+	uint16_t cmdif;
+
+	fw0 = mcx_rd(sc, MCX_FW_VER);
+	fw1 = mcx_rd(sc, MCX_CMDIF_FW_SUBVER);
+
+	printf(": FW %u.%u.%04u", MCX_FW_VER_MAJOR(fw0),
+	    MCX_FW_VER_MINOR(fw0), MCX_FW_VER_SUBMINOR(fw1));
+
+	cmdif = MCX_CMDIF(fw1);
+	if (cmdif != MCX_CMD_IF_SUPPORTED) {
+		printf(", unsupported command interface %u\n", cmdif);
+		return (-1);
+	}
+
+	return (0);
+}
+
+static int
+mcx_init_wait(struct mcx_softc *sc)
+{
+	unsigned int i;
+	uint32_t r;
+
+	for (i = 0; i < 2000; i++) {
+		r = mcx_rd(sc, MCX_STATE);
+		if ((r & MCX_STATE_MASK) == MCX_STATE_READY)
+			return (0);
+
+		delay(1000);
+		mcx_bar(sc, MCX_STATE, sizeof(uint32_t),
+		    BUS_SPACE_BARRIER_READ);
+	}
+
+	return (-1);
+}
+
+static uint8_t
+mcx_cmdq_poll(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, unsigned int msec)
+{
+	unsigned int i;
+
+	for (i = 0; i < msec; i++) {
+		bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem),
+		    0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_POSTRW);
+
+		if ((cqe->cq_status & MCX_CQ_STATUS_OWN_MASK) ==
+		    MCX_CQ_STATUS_OWN_SW) {
+			if (sc->sc_eqn != 0)
+				mcx_intr(sc);
+			return (0);
+		}
+
+		delay(1000);
+	}
+
+	return (ETIMEDOUT);
+}
+
+static uint32_t
+mcx_mix_u64(uint32_t xor, uint64_t u64)
+{
+	xor ^= u64 >> 32;
+	xor ^= u64;
+
+	return (xor);
+}
+
+static uint32_t
+mcx_mix_u32(uint32_t xor, uint32_t u32)
+{
+	xor ^= u32;
+
+	return (xor);
+}
+
+static uint32_t
+mcx_mix_u8(uint32_t xor, uint8_t u8)
+{
+	xor ^= u8;
+
+	return (xor);
+}
+
+static uint8_t
+mcx_mix_done(uint32_t xor)
+{
+	xor ^= xor >> 16;
+	xor ^= xor >> 8;
+
+	return (xor);
+}
+
+static uint8_t
+mcx_xor(const void *buf, size_t len)
+{
+	const uint32_t *dwords = buf;
+	uint32_t xor = 0xff;
+	size_t i;
+
+	len /= sizeof(*dwords);
+
+	for (i = 0; i < len; i++)
+		xor ^= dwords[i];
+
+	return (mcx_mix_done(xor));
+}
+
+static uint8_t
+mcx_cmdq_token(struct mcx_softc *sc)
+{
+	uint8_t token;
+
+	do {
+		token = ++sc->sc_cmdq_token;
+	} while (token == 0);
+
+	return (token);
+}
+
+static void
+mcx_cmdq_init(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
+    uint32_t ilen, uint32_t olen, uint8_t token)
+{
+	memset(cqe, 0, sc->sc_cmdq_size);
+
+	cqe->cq_type = MCX_CMDQ_TYPE_PCIE;
+	htobem32(&cqe->cq_input_length, ilen);
+	htobem32(&cqe->cq_output_length, olen);
+	cqe->cq_token = token;
+	cqe->cq_status = MCX_CQ_STATUS_OWN_HW;
+}
+
+static void
+mcx_cmdq_sign(struct mcx_cmdq_entry *cqe)
+{
+	cqe->cq_signature = ~mcx_xor(cqe, sizeof(*cqe));
+}
+
+static int
+mcx_cmdq_verify(const struct mcx_cmdq_entry *cqe)
+{
+	/* return (mcx_xor(cqe, sizeof(*cqe)) ? -1 :  0); */
+	return (0);
+}
+
+static void *
+mcx_cmdq_in(struct mcx_cmdq_entry *cqe)
+{
+	return (&cqe->cq_input_data);
+}
+
+static void *
+mcx_cmdq_out(struct mcx_cmdq_entry *cqe)
+{
+	return (&cqe->cq_output_data);
+}
+
+static void
+mcx_cmdq_post(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, unsigned int slot)
+{
+	mcx_cmdq_sign(cqe);
+
+	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem),
+	    0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_PRERW);
+
+	mcx_wr(sc, MCX_CMDQ_DOORBELL, 1U << slot);
+}
+
+static int
+mcx_enable_hca(struct mcx_softc *sc)
+{
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_cmd_enable_hca_in *in;
+	struct mcx_cmd_enable_hca_out *out;
+	int error;
+	uint8_t status;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_ENABLE_HCA);
+	in->cmd_op_mod = htobe16(0);
+	in->cmd_function_id = htobe16(0);
+
+	mcx_cmdq_post(sc, cqe, 0);
+
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf(", hca enable timeout\n");
+		return (-1);
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf(", hca enable command corrupt\n");
+		return (-1);
+	}
+
+	status = cqe->cq_output_data[0];
+	if (status != MCX_CQ_STATUS_OK) {
+		printf(", hca enable failed (%x)\n", status);
+		return (-1);
+	}
+
+	return (0);
+}
+
+static int
+mcx_teardown_hca(struct mcx_softc *sc, uint16_t profile)
+{
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_cmd_teardown_hca_in *in;
+	struct mcx_cmd_teardown_hca_out *out;
+	int error;
+	uint8_t status;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_TEARDOWN_HCA);
+	in->cmd_op_mod = htobe16(0);
+	in->cmd_profile = profile;
+
+	mcx_cmdq_post(sc, cqe, 0);
+
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf(", hca teardown timeout\n");
+		return (-1);
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf(", hca teardown command corrupt\n");
+		return (-1);
+	}
+
+	status = cqe->cq_output_data[0];
+	if (status != MCX_CQ_STATUS_OK) {
+		printf(", hca teardown failed (%x)\n", status);
+		return (-1);
+	}
+
+	return (0);
+}
+
+static int
+mcx_cmdq_mboxes_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm,
+    unsigned int nmb, uint64_t *ptr, uint8_t token)
+{
+	caddr_t kva;
+	uint64_t dva;
+	int i;
+	int error;
+
+	error = mcx_dmamem_alloc(sc, mxm,
+	    nmb * MCX_CMDQ_MAILBOX_SIZE, MCX_CMDQ_MAILBOX_ALIGN);
+	if (error != 0)
+		return (error);
+
+	mcx_dmamem_zero(mxm);
+
+	dva = MCX_DMA_DVA(mxm);
+	kva = MCX_DMA_KVA(mxm);
+	for (i = 0; i < nmb; i++) {
+		struct mcx_cmdq_mailbox *mbox = (struct mcx_cmdq_mailbox *)kva;
+
+		/* patch the cqe or mbox pointing at this one */
+		htobem64(ptr, dva);
+
+		/* fill in this mbox */
+		htobem32(&mbox->mb_block_number, i);
+		mbox->mb_token = token;
+
+		/* move to the next one */
+		ptr = &mbox->mb_next_ptr;
+
+		dva += MCX_CMDQ_MAILBOX_SIZE;
+		kva += MCX_CMDQ_MAILBOX_SIZE;
+	}
+
+	return (0);
+}
+
+static uint32_t
+mcx_cmdq_mbox_ctrl_sig(const struct mcx_cmdq_mailbox *mb)
+{
+	uint32_t xor = 0xff;
+
+	/* only 3 fields get set, so mix them directly */
+	xor = mcx_mix_u64(xor, mb->mb_next_ptr);
+	xor = mcx_mix_u32(xor, mb->mb_block_number);
+	xor = mcx_mix_u8(xor, mb->mb_token);
+
+	return (mcx_mix_done(xor));
+}
+
+static void
+mcx_cmdq_mboxes_sign(struct mcx_dmamem *mxm, unsigned int nmb)
+{
+	caddr_t kva;
+	int i;
+
+	kva = MCX_DMA_KVA(mxm);
+
+	for (i = 0; i < nmb; i++) {
+		struct mcx_cmdq_mailbox *mb = (struct mcx_cmdq_mailbox *)kva;
+		uint8_t sig = mcx_cmdq_mbox_ctrl_sig(mb);
+		mb->mb_ctrl_signature = sig;
+		mb->mb_signature = sig ^
+		    mcx_xor(mb->mb_data, sizeof(mb->mb_data));
+
+		kva += MCX_CMDQ_MAILBOX_SIZE;
+	}
+}
+
+static void
+mcx_cmdq_mboxes_sync(struct mcx_softc *sc, struct mcx_dmamem *mxm, int ops)
+{
+	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(mxm),
+	    0, MCX_DMA_LEN(mxm), ops);
+}
+
+static struct mcx_cmdq_mailbox *
+mcx_cq_mbox(struct mcx_dmamem *mxm, unsigned int i)
+{
+	caddr_t kva;
+
+	kva = MCX_DMA_KVA(mxm);
+	kva += i * MCX_CMDQ_MAILBOX_SIZE;
+
+	return ((struct mcx_cmdq_mailbox *)kva);
+}
+
+static inline void *
+mcx_cq_mbox_data(struct mcx_cmdq_mailbox *mb)
+{
+	return (&mb->mb_data);
+}
+
+static void
+mcx_cmdq_mboxes_copyin(struct mcx_dmamem *mxm, unsigned int nmb,
+    void *b, size_t len)
+{
+	caddr_t buf = b;
+	struct mcx_cmdq_mailbox *mb;
+	int i;
+
+	mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm);
+	for (i = 0; i < nmb; i++) {
+
+		memcpy(mb->mb_data, buf, min(sizeof(mb->mb_data), len));
+
+		if (sizeof(mb->mb_data) >= len)
+			break;
+
+		buf += sizeof(mb->mb_data);
+		len -= sizeof(mb->mb_data);
+		mb++;
+	}
+}
+
+static void
+mcx_cmdq_mboxes_copyout(struct mcx_dmamem *mxm, int nmb, void *b, size_t len)
+{
+	caddr_t buf = b;
+	struct mcx_cmdq_mailbox *mb;
+	int i;
+
+	mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm);
+	for (i = 0; i < nmb; i++) {
+		memcpy(buf, mb->mb_data, min(sizeof(mb->mb_data), len));
+
+		if (sizeof(mb->mb_data) >= len)
+			break;
+
+		buf += sizeof(mb->mb_data);
+		len -= sizeof(mb->mb_data);
+		mb++;
+	}
+}
+
+static void
+mcx_cq_mboxes_free(struct mcx_softc *sc, struct mcx_dmamem *mxm)
+{
+	mcx_dmamem_free(sc, mxm);
+}
+
+#if 0
+static void
+mcx_cmdq_dump(const struct mcx_cmdq_entry *cqe)
+{
+	unsigned int i;
+
+	printf(" type %02x, ilen %u, iptr %016llx", cqe->cq_type,
+	    bemtoh32(&cqe->cq_input_length), bemtoh64(&cqe->cq_input_ptr));
+
+	printf(", idata ");
+	for (i = 0; i < sizeof(cqe->cq_input_data); i++)
+		printf("%02x", cqe->cq_input_data[i]);
+
+	printf(", odata ");
+	for (i = 0; i < sizeof(cqe->cq_output_data); i++)
+		printf("%02x", cqe->cq_output_data[i]);
+
+	printf(", optr %016llx, olen %u, token %02x, sig %02x, status %02x",
+	    bemtoh64(&cqe->cq_output_ptr), bemtoh32(&cqe->cq_output_length),
+	    cqe->cq_token, cqe->cq_signature, cqe->cq_status);
+}
+
+static void
+mcx_cmdq_mbox_dump(struct mcx_dmamem *mboxes, int num)
+{
+	int i, j;
+	uint8_t *d;
+
+	for (i = 0; i < num; i++) {
+		struct mcx_cmdq_mailbox *mbox;
+		mbox = mcx_cq_mbox(mboxes, i);
+
+		d = mcx_cq_mbox_data(mbox);
+		for (j = 0; j < MCX_CMDQ_MAILBOX_DATASIZE; j++) {
+			if (j != 0 && (j % 16 == 0))
+				printf("\n");
+			printf("%.2x ", d[j]);
+		}
+	}
+}
+#endif
+
+static int
+mcx_read_hca_reg(struct mcx_softc *sc, uint16_t reg, void *data, int len)
+{
+	struct mcx_dmamem mxm;
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_cmd_access_reg_in *in;
+	struct mcx_cmd_access_reg_out *out;
+	uint8_t token = mcx_cmdq_token(sc);
+	int error, nmb;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	mcx_cmdq_init(sc, cqe, sizeof(*in) + len, sizeof(*out) + len,
+	    token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_ACCESS_REG);
+	in->cmd_op_mod = htobe16(MCX_REG_OP_READ);
+	in->cmd_register_id = htobe16(reg);
+
+	nmb = howmany(len, MCX_CMDQ_MAILBOX_DATASIZE);
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb, &cqe->cq_output_ptr, token) != 0) {
+		printf(", unable to allocate access reg mailboxen\n");
+		return (-1);
+	}
+	cqe->cq_input_ptr = cqe->cq_output_ptr;
+	mcx_cmdq_mboxes_copyin(&mxm, nmb, data, len);
+	mcx_cmdq_mboxes_sign(&mxm, nmb);
+	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);
+
+	mcx_cmdq_post(sc, cqe, 0);
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);
+
+	if (error != 0) {
+		printf("%s: access reg (read %x) timeout\n", DEVNAME(sc), reg);
+		goto free;
+	}
+	error = mcx_cmdq_verify(cqe);
+	if (error != 0) {
+		printf("%s: access reg (read %x) reply corrupt\n", DEVNAME(sc), reg);
+		goto free;
+	}
+
+	out = mcx_cmdq_out(cqe);
+	if (out->cmd_status != MCX_CQ_STATUS_OK) {
+		printf("%s: access reg (read %x) failed (%x, %.6x)\n", DEVNAME(sc), reg, out->cmd_status, out->cmd_syndrome);
+		error = -1;
+		goto free;
+	}
+
+	mcx_cmdq_mboxes_copyout(&mxm, nmb, data, len);
+free:
+	mcx_dmamem_free(sc, &mxm);
+
+	return (error);
+}
+
+#if 0
+static int
+mcx_write_hca_reg(struct mcx_softc *sc, uint16_t reg, const uint8_t *data, int len)
+{
+	return (0);
+}
+#endif
+
+static int
+mcx_set_issi(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, unsigned int slot)
+{
+	struct mcx_cmd_set_issi_in *in;
+	struct mcx_cmd_set_issi_out *out;
+	uint8_t status;
+
+	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_QUERY_ISSI);
+	in->cmd_op_mod = htobe16(0);
+	in->cmd_current_issi = htobe16(MCX_ISSI);
+
+	mcx_cmdq_post(sc, cqe, slot);
+	if (mcx_cmdq_poll(sc, cqe, 1000) != 0)
+		return (-1);
+	if (mcx_cmdq_verify(cqe) != 0)
+		return (-1);
+
+	status = cqe->cq_output_data[0];
+	if (status != MCX_CQ_STATUS_OK)
+		return (-1);
+
+	return (0);
+}
+
+static int
+mcx_issi(struct mcx_softc *sc)
+{
+	struct mcx_dmamem mxm;
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_cmd_query_issi_in *in;
+	struct mcx_cmd_query_issi_il_out *out;
+	struct mcx_cmd_query_issi_mb_out *mb;
+	uint8_t token = mcx_cmdq_token(sc);
+	uint8_t status;
+	int error;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mb), token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_QUERY_ISSI);
+	in->cmd_op_mod = htobe16(0);
+
+	CTASSERT(sizeof(*mb) <= MCX_CMDQ_MAILBOX_DATASIZE);
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
+	    &cqe->cq_output_ptr, token) != 0) {
+		printf(", unable to allocate query issi mailbox\n");
+		return (-1);
+	}
+	mcx_cmdq_mboxes_sign(&mxm, 1);
+
+	mcx_cmdq_post(sc, cqe, 0);
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf(", query issi timeout\n");
+		goto free;
+	}
+	error = mcx_cmdq_verify(cqe);
+	if (error != 0) {
+		printf(", query issi reply corrupt\n");
+		goto free;
+	}
+
+	status = cqe->cq_output_data[0];
+	switch (status) {
+	case MCX_CQ_STATUS_OK:
+		break;
+	case MCX_CQ_STATUS_BAD_OPCODE:
+		/* use ISSI 0 */
+		goto free;
+	default:
+		printf(", query issi failed (%x)\n", status);
+		error = -1;
+		goto free;
+	}
+
+	out = mcx_cmdq_out(cqe);
+	if (out->cmd_current_issi == htobe16(MCX_ISSI)) {
+		/* use ISSI 1 */
+		goto free;
+	}
+
+	/* don't need to read cqe anymore, can be used for SET ISSI */
+
+	mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
+	CTASSERT(MCX_ISSI < NBBY);
+	 /* XXX math is hard */
+	if (!ISSET(mb->cmd_supported_issi[79], 1 << MCX_ISSI)) {
+		/* use ISSI 0 */
+		goto free;
+	}
+
+	if (mcx_set_issi(sc, cqe, 0) != 0) {
+		/* ignore the error, just use ISSI 0 */
+	} else {
+		/* use ISSI 1 */
+	}
+
+free:
+	mcx_cq_mboxes_free(sc, &mxm);
+	return (error);
+}
+
+static int
+mcx_query_pages(struct mcx_softc *sc, uint16_t type,
+    uint32_t *npages, uint16_t *func_id)
+{
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_cmd_query_pages_in *in;
+	struct mcx_cmd_query_pages_out *out;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_QUERY_PAGES);
+	in->cmd_op_mod = type;
+
+	mcx_cmdq_post(sc, cqe, 0);
+	if (mcx_cmdq_poll(sc, cqe, 1000) != 0) {
+		printf(", query pages timeout\n");
+		return (-1);
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf(", query pages reply corrupt\n");
+		return (-1);
+	}
+
+	out = mcx_cmdq_out(cqe);
+	if (out->cmd_status != MCX_CQ_STATUS_OK) {
+		printf(", query pages failed (%x)\n", out->cmd_status);
+		return (-1);
+	}
+
+	*func_id = out->cmd_func_id;
+	*npages = bemtoh32(&out->cmd_num_pages);
+
+	return (0);
+}
+
+struct bus_dma_iter {
+	bus_dmamap_t		i_map;
+	bus_size_t		i_offset;
+	unsigned int		i_index;
+};
+
+static void
+bus_dma_iter_init(struct bus_dma_iter *i, bus_dmamap_t map)
+{
+	i->i_map = map;
+	i->i_offset = 0;
+	i->i_index = 0;
+}
+
+static bus_addr_t
+bus_dma_iter_addr(struct bus_dma_iter *i)
+{
+	return (i->i_map->dm_segs[i->i_index].ds_addr + i->i_offset);
+}
+
+static void
+bus_dma_iter_add(struct bus_dma_iter *i, bus_size_t size)
+{
+	bus_dma_segment_t *seg = i->i_map->dm_segs + i->i_index;
+	bus_size_t diff;
+
+	do {
+		diff = seg->ds_len - i->i_offset;
+		if (size < diff)
+			break;
+
+		size -= diff;
+
+		seg++;
+
+		i->i_offset = 0;
+		i->i_index++;
+	} while (size > 0);
+
+	i->i_offset += size;
+}
+
+static int
+mcx_add_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t func_id)
+{
+	struct mcx_dmamem mxm;
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_cmd_manage_pages_in *in;
+	struct mcx_cmd_manage_pages_out *out;
+	unsigned int paslen, nmb, i, j, npages;
+	struct bus_dma_iter iter;
+	uint64_t *pas;
+	uint8_t status;
+	uint8_t token = mcx_cmdq_token(sc);
+	int error;
+
+	npages = mhm->mhm_npages;
+
+	paslen = sizeof(*pas) * npages;
+	nmb = howmany(paslen, MCX_CMDQ_MAILBOX_DATASIZE);
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	mcx_cmdq_init(sc, cqe, sizeof(*in) + paslen, sizeof(*out), token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_MANAGE_PAGES);
+	in->cmd_op_mod = htobe16(MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS);
+	in->cmd_func_id = func_id;
+	htobem32(&in->cmd_input_num_entries, npages);
+
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb,
+	    &cqe->cq_input_ptr, token) != 0) {
+		printf(", unable to allocate manage pages mailboxen\n");
+		return (-1);
+	}
+
+	bus_dma_iter_init(&iter, mhm->mhm_map);
+	for (i = 0; i < nmb; i++) {
+		unsigned int lim;
+
+		pas = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, i));
+		lim = min(MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas), npages);
+
+		for (j = 0; j < lim; j++) {
+			htobem64(&pas[j], bus_dma_iter_addr(&iter));
+			bus_dma_iter_add(&iter, MCX_PAGE_SIZE);
+		}
+
+		npages -= lim;
+	}
+
+	mcx_cmdq_mboxes_sign(&mxm, nmb);
+
+	mcx_cmdq_post(sc, cqe, 0);
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf(", manage pages timeout\n");
+		goto free;
+	}
+	error = mcx_cmdq_verify(cqe);
+	if (error != 0) {
+		printf(", manage pages reply corrupt\n");
+		goto free;
+	}
+
+	status = cqe->cq_output_data[0];
+	if (status != MCX_CQ_STATUS_OK) {
+		printf(", manage pages failed (%x)\n", status);
+		error = -1;
+		goto free;
+	}
+
+free:
+	mcx_dmamem_free(sc, &mxm);
+
+	return (error);
+}
+
+static int
+mcx_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t type)
+{
+	uint32_t npages;
+	uint16_t func_id;
+
+	if (mcx_query_pages(sc, type, &npages, &func_id) != 0) {
+		/* error printed by mcx_query_pages */
+		return (-1);
+	}
+
+	if (npages == 0)
+		return (0);
+
+	if (mcx_hwmem_alloc(sc, mhm, npages) != 0) {
+		printf(", unable to allocate hwmem\n");
+		return (-1);
+	}
+
+	if (mcx_add_pages(sc, mhm, func_id) != 0) {
+		printf(", unable to add hwmem\n");
+		goto free;
+	}
+
+	return (0);
+
+free:
+	mcx_hwmem_free(sc, mhm);
+
+	return (-1);
+}
+
+static int
+mcx_hca_max_caps(struct mcx_softc *sc)
+{
+	struct mcx_dmamem mxm;
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_cmd_query_hca_cap_in *in;
+	struct mcx_cmd_query_hca_cap_out *out;
+	struct mcx_cmdq_mailbox *mb;
+	struct mcx_cap_device *hca = mcx_cq_mbox_data(mb);
+	uint8_t status;
+	uint8_t token = mcx_cmdq_token(sc);
+	int error;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN,
+	    token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP);
+	in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_MAX |
+	    MCX_CMD_QUERY_HCA_CAP_DEVICE);
+
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES,
+	    &cqe->cq_output_ptr, token) != 0) {
+		printf(", unable to allocate query hca caps mailboxen\n");
+		return (-1);
+	}
+	mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES);
+	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);
+
+	mcx_cmdq_post(sc, cqe, 0);
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);
+
+	if (error != 0) {
+		printf(", query hca caps timeout\n");
+		goto free;
+	}
+	error = mcx_cmdq_verify(cqe);
+	if (error != 0) {
+		printf(", query hca caps reply corrupt\n");
+		goto free;
+	}
+
+	status = cqe->cq_output_data[0];
+	if (status != MCX_CQ_STATUS_OK) {
+		printf(", query hca caps failed (%x)\n", status);
+		error = -1;
+		goto free;
+	}
+
+	mb = mcx_cq_mbox(&mxm, 0);
+	hca = mcx_cq_mbox_data(mb);
+
+	if (hca->log_pg_sz > PAGE_SHIFT) {
+		printf(", minimum system page shift %u is too large\n",
+		    hca->log_pg_sz);
+		error = -1;
+		goto free;
+	}
+	/*
+	 * blueflame register is split into two buffers, and we must alternate
+	 * between the two of them.
+	 */
+	sc->sc_bf_size = (1 << hca->log_bf_reg_size) / 2;
+
+free:
+	mcx_dmamem_free(sc, &mxm);
+
+	return (error);
+}
+
+static int
+mcx_hca_set_caps(struct mcx_softc *sc)
+{
+	struct mcx_dmamem mxm;
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_cmd_query_hca_cap_in *in;
+	struct mcx_cmd_query_hca_cap_out *out;
+	struct mcx_cmdq_mailbox *mb;
+	struct mcx_cap_device *hca = mcx_cq_mbox_data(mb);
+	uint8_t status;
+	uint8_t token = mcx_cmdq_token(sc);
+	int error;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN,
+	    token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP);
+	in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_CURRENT |
+	    MCX_CMD_QUERY_HCA_CAP_DEVICE);
+
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES,
+	    &cqe->cq_output_ptr, token) != 0) {
+		printf(", unable to allocate manage pages mailboxen\n");
+		return (-1);
+	}
+	mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES);
+	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);
+
+	mcx_cmdq_post(sc, cqe, 0);
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);
+
+	if (error != 0) {
+		printf(", query hca caps timeout\n");
+		goto free;
+	}
+	error = mcx_cmdq_verify(cqe);
+	if (error != 0) {
+		printf(", query hca caps reply corrupt\n");
+		goto free;
+	}
+
+	status = cqe->cq_output_data[0];
+	if (status != MCX_CQ_STATUS_OK) {
+		printf(", query hca caps failed (%x)\n", status);
+		error = -1;
+		goto free;
+	}
+
+	mb = mcx_cq_mbox(&mxm, 0);
+	hca = mcx_cq_mbox_data(mb);
+
+	hca->log_pg_sz = PAGE_SHIFT;
+
+free:
+	mcx_dmamem_free(sc, &mxm);
+
+	return (error);
+}
+
+
+static int
+mcx_init_hca(struct mcx_softc *sc)
+{
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_cmd_init_hca_in *in;
+	struct mcx_cmd_init_hca_out *out;
+	int error;
+	uint8_t status;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_INIT_HCA);
+	in->cmd_op_mod = htobe16(0);
+
+	mcx_cmdq_post(sc, cqe, 0);
+
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf(", hca init timeout\n");
+		return (-1);
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf(", hca init command corrupt\n");
+		return (-1);
+	}
+
+	status = cqe->cq_output_data[0];
+	if (status != MCX_CQ_STATUS_OK) {
+		printf(", hca init failed (%x)\n", status);
+		return (-1);
+	}
+
+	return (0);
+}
+
+static int
+mcx_set_driver_version(struct mcx_softc *sc)
+{
+	struct mcx_dmamem mxm;
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_cmd_set_driver_version_in *in;
+	struct mcx_cmd_set_driver_version_out *out;
+	int error;
+	int token;
+	uint8_t status;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	token = mcx_cmdq_token(sc);
+	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(struct mcx_cmd_set_driver_version),
+	    sizeof(*out), token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_SET_DRIVER_VERSION);
+	in->cmd_op_mod = htobe16(0);
+
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
+	    &cqe->cq_input_ptr, token) != 0) {
+		printf(", unable to allocate set driver version mailboxen\n");
+		return (-1);
+	}
+	strlcpy(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)),
+	    "OpenBSD,mcx,1.000.000000", MCX_CMDQ_MAILBOX_DATASIZE);
+
+	mcx_cmdq_mboxes_sign(&mxm, 1);
+	mcx_cmdq_post(sc, cqe, 0);
+
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf(", set driver version timeout\n");
+		goto free;
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf(", set driver version command corrupt\n");
+		goto free;
+	}
+
+	status = cqe->cq_output_data[0];
+	if (status != MCX_CQ_STATUS_OK) {
+		printf(", set driver version failed (%x)\n", status);
+		error = -1;
+		goto free;
+	}
+
+free:
+	mcx_dmamem_free(sc, &mxm);
+
+	return (error);
+}
+
+static int
+mcx_iff(struct mcx_softc *sc)
+{
+	struct mcx_dmamem mxm;
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_cmd_modify_nic_vport_context_in *in;
+	struct mcx_cmd_modify_nic_vport_context_out *out;
+	struct mcx_nic_vport_ctx *ctx;
+	int error;
+	int token;
+	int insize;
+
+	/* calculate size of mcast address list */
+	insize = sizeof(struct mcx_nic_vport_ctx) + 240;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	token = mcx_cmdq_token(sc);
+	mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_MODIFY_NIC_VPORT_CONTEXT);
+	in->cmd_op_mod = htobe16(0);
+	in->cmd_field_select = htobe32(
+	    MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC |
+	    MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU);
+	    /* add address list when we do mcast */
+
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) {
+		printf(", unable to allocate modify nic vport context mailboxen\n");
+		return (-1);
+	}
+	ctx = (struct mcx_nic_vport_ctx *)(((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 240);
+	ctx->vp_mtu = htobe32(MCX_HARDMTU);
+	ctx->vp_flags = htobe16(MCX_NIC_VPORT_CTX_PROMISC_UCAST |
+	    MCX_NIC_VPORT_CTX_PROMISC_MCAST |
+	    MCX_NIC_VPORT_CTX_PROMISC_ALL);
+
+	mcx_cmdq_mboxes_sign(&mxm, 1);
+	mcx_cmdq_post(sc, cqe, 0);
+
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf(", modify nic vport context timeout\n");
+		goto free;
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf(", modify nic vport context command corrupt\n");
+		goto free;
+	}
+
+	out = mcx_cmdq_out(cqe);
+	if (out->cmd_status != MCX_CQ_STATUS_OK) {
+		printf(", modify nic vport context failed (%x, %x)\n", out->cmd_status, out->cmd_syndrome);
+		error = -1;
+		goto free;
+	}
+
+free:
+	mcx_dmamem_free(sc, &mxm);
+
+	return (error);
+}
+
+static int
+mcx_alloc_uar(struct mcx_softc *sc)
+{
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_cmd_alloc_uar_in *in;
+	struct mcx_cmd_alloc_uar_out *out;
+	int error;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_ALLOC_UAR);
+	in->cmd_op_mod = htobe16(0);
+
+	mcx_cmdq_post(sc, cqe, 0);
+
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf(", alloc uar timeout\n");
+		return (-1);
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf(", alloc uar command corrupt\n");
+		return (-1);
+	}
+
+	out = mcx_cmdq_out(cqe);
+	if (out->cmd_status != MCX_CQ_STATUS_OK) {
+		printf(", alloc uar failed (%x)\n", out->cmd_status);
+		return (-1);
+	}
+
+	sc->sc_uar = betoh32(out->cmd_uar);
+
+	return (0);
+}
+
+static int
+mcx_create_eq(struct mcx_softc *sc)
+{
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_dmamem mxm;
+	struct mcx_cmd_create_eq_in *in;
+	struct mcx_cmd_create_eq_mb_in *mbin;
+	struct mcx_cmd_create_eq_out *out;
+	struct mcx_eq_entry *eqe;
+	int error;
+	uint64_t *pas;
+	int insize, npages, paslen, i, token;
+
+	sc->sc_eq_cons = 0;
+
+	npages = howmany((1 << MCX_LOG_EQ_SIZE) * sizeof(struct mcx_eq_entry), PAGE_SIZE);
+	paslen = npages * sizeof(*pas);
+	insize = sizeof(struct mcx_cmd_create_eq_mb_in) + paslen;
+
+	if (mcx_dmamem_alloc(sc, &sc->sc_eq_mem, npages * MCX_PAGE_SIZE,
+	    MCX_PAGE_SIZE) != 0) {
+		printf(", unable to allocate event queue memory\n");
+		return (-1);
+	}
+
+	eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&sc->sc_eq_mem);
+	for (i = 0; i < (1 << MCX_LOG_EQ_SIZE); i++) {
+		eqe[i].eq_owner = MCX_EQ_ENTRY_OWNER_INIT;
+	}
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	token = mcx_cmdq_token(sc);
+	mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_CREATE_EQ);
+	in->cmd_op_mod = htobe16(0);
+
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
+	    &cqe->cq_input_ptr, token) != 0) {
+		printf(", unable to allocate create eq mailboxen\n");
+		return (-1);
+	}
+	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
+	mbin->cmd_eq_ctx.eq_uar_size = htobe32((MCX_LOG_EQ_SIZE << MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT)
+	    | sc->sc_uar);
+	mbin->cmd_event_bitmask = htobe64(
+	    (1ull << MCX_EVENT_TYPE_INTERNAL_ERROR) |
+	    (1ull << MCX_EVENT_TYPE_PORT_CHANGE) |
+	    (1ull << MCX_EVENT_TYPE_CMD_COMPLETION) |
+	    (1ull << MCX_EVENT_TYPE_PAGE_REQUEST));
+
+	/* physical addresses follow the mailbox in data */
+	pas = (uint64_t *)(mbin + 1);
+	for (i = 0; i < npages; i++) {
+		pas[i] = htobe64(MCX_DMA_DVA(&sc->sc_eq_mem) + (i * MCX_PAGE_SIZE));
+	}
+	mcx_cmdq_mboxes_sign(&mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE));
+	mcx_cmdq_post(sc, cqe, 0);
+
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf(", create eq timeout\n");
+		goto free;
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf(", create eq command corrupt\n");
+		goto free;
+	}
+
+	out = mcx_cmdq_out(cqe);
+	if (out->cmd_status != MCX_CQ_STATUS_OK) {
+		printf(", create eq failed (%x, %x)\n", out->cmd_status, betoh32(out->cmd_syndrome));
+		error = -1;
+		goto free;
+	}
+
+	sc->sc_eqn = betoh32(out->cmd_eqn);
+	mcx_arm_eq(sc);
+free:
+	mcx_dmamem_free(sc, &mxm);
+	return (error);
+}
+
+static int
+mcx_alloc_pd(struct mcx_softc *sc)
+{
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_cmd_alloc_pd_in *in;
+	struct mcx_cmd_alloc_pd_out *out;
+	int error;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_ALLOC_PD);
+	in->cmd_op_mod = htobe16(0);
+
+	mcx_cmdq_post(sc, cqe, 0);
+
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf(", alloc pd timeout\n");
+		return (-1);
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf(", alloc pd command corrupt\n");
+		return (-1);
+	}
+
+	out = mcx_cmdq_out(cqe);
+	if (out->cmd_status != MCX_CQ_STATUS_OK) {
+		printf(", alloc pd failed (%x)\n", out->cmd_status);
+		return (-1);
+	}
+
+	sc->sc_pd = betoh32(out->cmd_pd);
+	return (0);
+}
+
+static int
+mcx_alloc_tdomain(struct mcx_softc *sc)
+{
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_cmd_alloc_td_in *in;
+	struct mcx_cmd_alloc_td_out *out;
+	int error;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_ALLOC_TRANSPORT_DOMAIN);
+	in->cmd_op_mod = htobe16(0);
+
+	mcx_cmdq_post(sc, cqe, 0);
+
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf(", alloc transport domain timeout\n");
+		return (-1);
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf(", alloc transport domain command corrupt\n");
+		return (-1);
+	}
+
+	out = mcx_cmdq_out(cqe);
+	if (out->cmd_status != MCX_CQ_STATUS_OK) {
+		printf(", alloc transport domain failed (%x)\n", out->cmd_status);
+		return (-1);
+	}
+
+	sc->sc_tdomain = betoh32(out->cmd_tdomain);
+	return (0);
+}
+
+static int
+mcx_query_nic_vport_context(struct mcx_softc *sc)
+{
+	struct mcx_dmamem mxm;
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_cmd_query_nic_vport_context_in *in;
+	struct mcx_cmd_query_nic_vport_context_out *out;
+	struct mcx_nic_vport_ctx *ctx;
+	uint8_t *addr;
+	int error, token, i;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	token = mcx_cmdq_token(sc);
+	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx), token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_QUERY_NIC_VPORT_CONTEXT);
+	in->cmd_op_mod = htobe16(0);
+	in->cmd_allowed_list_type = 0;
+
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_output_ptr, token) != 0) {
+		printf(", unable to allocate query nic vport context mailboxen\n");
+		return (-1);
+	}
+	mcx_cmdq_mboxes_sign(&mxm, 1);
+	mcx_cmdq_post(sc, cqe, 0);
+
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf(", query nic vport context timeout\n");
+		goto free;
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf(", query nic vport context command corrupt\n");
+		goto free;
+	}
+
+	out = mcx_cmdq_out(cqe);
+	if (out->cmd_status != MCX_CQ_STATUS_OK) {
+		printf(", query nic vport context failed (%x, %x)\n", out->cmd_status, out->cmd_syndrome);
+		error = -1;
+		goto free;
+	}
+
+	ctx = (struct mcx_nic_vport_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
+	addr = (uint8_t *)&ctx->vp_perm_addr;
+	for (i = 0; i < ETHER_ADDR_LEN; i++) {
+		sc->sc_ac.ac_enaddr[i] = addr[i + 2];
+	}
+free:
+	mcx_dmamem_free(sc, &mxm);
+
+	return (error);
+}
+
+static int
+mcx_query_special_contexts(struct mcx_softc *sc)
+{
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_cmd_query_special_ctx_in *in;
+	struct mcx_cmd_query_special_ctx_out *out;
+	int error;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_QUERY_SPECIAL_CONTEXTS);
+	in->cmd_op_mod = htobe16(0);
+
+	mcx_cmdq_post(sc, cqe, 0);
+
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf(", query special contexts timeout\n");
+		return (-1);
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf(", query special contexts command corrupt\n");
+		return (-1);
+	}
+
+	out = mcx_cmdq_out(cqe);
+	if (out->cmd_status != MCX_CQ_STATUS_OK) {
+		printf(", query special contexts failed (%x)\n", out->cmd_status);
+		return (-1);
+	}
+
+	sc->sc_lkey = betoh32(out->cmd_resd_lkey);
+	return (0);
+}
+
+static int
+mcx_create_cq(struct mcx_softc *sc, int eqn)
+{
+	struct mcx_cmdq_entry *cmde;
+	struct mcx_cq_entry *cqe;
+	struct mcx_cq *cq;
+	struct mcx_dmamem mxm;
+	struct mcx_cmd_create_cq_in *in;
+	struct mcx_cmd_create_cq_mb_in *mbin;
+	struct mcx_cmd_create_cq_out *out;
+	int error;
+	uint64_t *pas;
+	int insize, npages, paslen, i, token;
+
+	if (sc->sc_num_cq >= MCX_MAX_CQS) {
+		printf("%s: tried to create too many cqs\n", DEVNAME(sc));
+		return (-1);
+	}
+	cq = &sc->sc_cq[sc->sc_num_cq];
+
+	npages = howmany((1 << MCX_LOG_CQ_SIZE) * sizeof(struct mcx_cq_entry), PAGE_SIZE);
+	paslen = npages * sizeof(*pas);
+	insize = sizeof(struct mcx_cmd_create_cq_mb_in) + paslen;
+
+	if (mcx_dmamem_alloc(sc, &cq->cq_mem, npages * MCX_PAGE_SIZE,
+	    MCX_PAGE_SIZE) != 0) {
+		printf("%s: unable to allocate completion queue memory\n", DEVNAME(sc));
+		return (-1);
+	}
+	cqe = MCX_DMA_KVA(&cq->cq_mem);
+	for (i = 0; i < (1 << MCX_LOG_CQ_SIZE); i++) {
+		cqe[i].cq_opcode_owner = MCX_CQ_ENTRY_FLAG_OWNER;
+	}
+
+	cmde = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	token = mcx_cmdq_token(sc);
+	mcx_cmdq_init(sc, cmde, sizeof(*in) + insize, sizeof(*out), token);
+
+	in = mcx_cmdq_in(cmde);
+	in->cmd_opcode = htobe16(MCX_CMD_CREATE_CQ);
+	in->cmd_op_mod = htobe16(0);
+
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
+	    &cmde->cq_input_ptr, token) != 0) {
+		printf("%s: unable to allocate create cq mailboxen\n", DEVNAME(sc));
+		error = -1;
+		goto free;
+	}
+	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
+	mbin->cmd_cq_ctx.cq_uar_size = htobe32((MCX_LOG_CQ_SIZE << MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT)
+	    | sc->sc_uar);
+	mbin->cmd_cq_ctx.cq_eqn = htobe32(eqn);
+	/* set event moderation bits here */
+	mbin->cmd_cq_ctx.cq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) +
+	    MCX_CQ_DOORBELL_OFFSET + (MCX_CQ_DOORBELL_SIZE * sc->sc_num_cq));
+
+	/* physical addresses follow the mailbox in data */
+	pas = (uint64_t *)(mbin + 1);
+	for (i = 0; i < npages; i++) {
+		pas[i] = htobe64(MCX_DMA_DVA(&cq->cq_mem) + (i * MCX_PAGE_SIZE));
+	}
+	mcx_cmdq_post(sc, cmde, 0);
+
+	error = mcx_cmdq_poll(sc, cmde, 1000);
+	if (error != 0) {
+		printf("%s: create cq timeout\n", DEVNAME(sc));
+		goto free;
+	}
+	if (mcx_cmdq_verify(cmde) != 0) {
+		printf("%s: create cq command corrupt\n", DEVNAME(sc));
+		goto free;
+	}
+
+	out = mcx_cmdq_out(cmde);
+	if (out->cmd_status != MCX_CQ_STATUS_OK) {
+		printf("%s: create cq failed (%x, %x)\n", DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
+		error = -1;
+		goto free;
+	}
+
+	cq->cq_n = betoh32(out->cmd_cqn);
+	cq->cq_cons = 0;
+	cq->cq_count = 0;
+	cq->cq_doorbell = MCX_DMA_KVA(&sc->sc_doorbell_mem) + MCX_CQ_DOORBELL_OFFSET +
+	    (MCX_CQ_DOORBELL_SIZE * sc->sc_num_cq);
+	mcx_arm_cq(sc, cq);
+	sc->sc_num_cq++;
+
+free:
+	mcx_dmamem_free(sc, &mxm);
+	return (error);
+}
+
+static int
+mcx_create_rq(struct mcx_softc *sc, int cqn)
+{
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_dmamem mxm;
+	struct mcx_cmd_create_rq_in *in;
+	struct mcx_cmd_create_rq_out *out;
+	struct mcx_rq_ctx *mbin;
+	int error;
+	uint64_t *pas;
+	uint8_t *doorbell;
+	int insize, npages, paslen, i, token;
+
+	npages = howmany((1 << MCX_LOG_RQ_SIZE) * sizeof(struct mcx_rq_entry), PAGE_SIZE);
+	paslen = npages * sizeof(*pas);
+	insize = 0x10 + sizeof(struct mcx_rq_ctx) + paslen;
+
+	if (mcx_dmamem_alloc(sc, &sc->sc_rq_mem, npages * MCX_PAGE_SIZE,
+	    MCX_PAGE_SIZE) != 0) {
+		printf("%s: unable to allocate receive queue memory\n", DEVNAME(sc));
+		return (-1);
+	}
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	token = mcx_cmdq_token(sc);
+	mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQ);
+	in->cmd_op_mod = htobe16(0);
+
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
+	    &cqe->cq_input_ptr, token) != 0) {
+		printf("%s: unable to allocate create rq mailboxen\n", DEVNAME(sc));
+		error = -1;
+		goto free;
+	}
+	mbin = (struct mcx_rq_ctx *)(((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10);
+	mbin->rq_flags = htobe32(MCX_RQ_CTX_RLKEY | MCX_RQ_CTX_VLAN_STRIP_DIS);
+	mbin->rq_cqn = htobe32(cqn);
+	mbin->rq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC;
+	mbin->rq_wq.wq_pd = htobe32(sc->sc_pd);
+	mbin->rq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) +
+	    MCX_RQ_DOORBELL_OFFSET);
+	mbin->rq_wq.wq_log_stride = htobe16(4);
+	mbin->rq_wq.wq_log_size = MCX_LOG_RQ_SIZE;
+
+	/* physical addresses follow the mailbox in data */
+	pas = (uint64_t *)(mbin + 1);
+	for (i = 0; i < npages; i++) {
+		pas[i] = htobe64(MCX_DMA_DVA(&sc->sc_rq_mem) + (i * MCX_PAGE_SIZE));
+	}
+	mcx_cmdq_post(sc, cqe, 0);
+
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf("%s: create rq timeout\n", DEVNAME(sc));
+		goto free;
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf("%s: create rq command corrupt\n", DEVNAME(sc));
+		goto free;
+	}
+
+	out = mcx_cmdq_out(cqe);
+	if (out->cmd_status != MCX_CQ_STATUS_OK) {
+		printf("%s: create rq failed (%x, %x)\n", DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
+		error = -1;
+		goto free;
+	}
+
+	sc->sc_rqn = betoh32(out->cmd_rqn);
+
+	doorbell = MCX_DMA_KVA(&sc->sc_doorbell_mem);
+	sc->sc_rx_doorbell = (uint32_t *)(doorbell + MCX_RQ_DOORBELL_OFFSET);
+
+free:
+	mcx_dmamem_free(sc, &mxm);
+	return (error);
+}
+
+static int
+mcx_ready_rq(struct mcx_softc *sc)
+{
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_dmamem mxm;
+	struct mcx_cmd_modify_rq_in *in;
+	struct mcx_cmd_modify_rq_mb_in *mbin;
+	struct mcx_cmd_modify_rq_out *out;
+	int error;
+	int token;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	token = mcx_cmdq_token(sc);
+	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_MODIFY_RQ);
+	in->cmd_op_mod = htobe16(0);
+	in->cmd_rq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | sc->sc_rqn);
+
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) {
+		printf("%s: unable to allocate modify rq mailbox\n", DEVNAME(sc));
+		return (-1);
+	}
+	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
+	mbin->cmd_rq_ctx.rq_flags = htobe32(MCX_QUEUE_STATE_RDY << MCX_RQ_CTX_STATE_SHIFT);
+
+	mcx_cmdq_mboxes_sign(&mxm, 1);
+	mcx_cmdq_post(sc, cqe, 0);
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf("%s: modify rq timeout\n", DEVNAME(sc));
+		goto free;
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf("%s: modify rq command corrupt\n", DEVNAME(sc));
+		goto free;
+	}
+
+	out = mcx_cmdq_out(cqe);
+	if (out->cmd_status != MCX_CQ_STATUS_OK) {
+		printf("%s: modify rq failed (%x, %x)\n", DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
+		error = -1;
+		goto free;
+	}
+
+free:
+	mcx_dmamem_free(sc, &mxm);
+	return (error);
+}
+
+static int
+mcx_create_tir(struct mcx_softc *sc)
+{
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_dmamem mxm;
+	struct mcx_cmd_create_tir_in *in;
+	struct mcx_cmd_create_tir_mb_in *mbin;
+	struct mcx_cmd_create_tir_out *out;
+	int error;
+	int token;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	token = mcx_cmdq_token(sc);
+	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR);
+	in->cmd_op_mod = htobe16(0);
+
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) {
+		printf("%s: unable to allocate create tir mailbox\n", DEVNAME(sc));
+		return (-1);
+	}
+	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
+	/* leave disp_type = 0, so packets get sent to the inline rqn */
+	mbin->cmd_inline_rqn = htobe32(sc->sc_rqn);
+	mbin->cmd_tdomain = htobe32(sc->sc_tdomain);
+
+	mcx_cmdq_post(sc, cqe, 0);
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf("%s: create tir timeout\n", DEVNAME(sc));
+		goto free;
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf("%s: create tir command corrupt\n", DEVNAME(sc));
+		goto free;
+	}
+
+	out = mcx_cmdq_out(cqe);
+	if (out->cmd_status != MCX_CQ_STATUS_OK) {
+		printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
+		error = -1;
+		goto free;
+	}
+
+	sc->sc_tirn = betoh32(out->cmd_tirn);
+free:
+	mcx_dmamem_free(sc, &mxm);
+	return (error);
+}
+
+static int
+mcx_create_sq(struct mcx_softc *sc, int cqn)
+{
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_dmamem mxm;
+	struct mcx_cmd_create_sq_in *in;
+	struct mcx_sq_ctx *mbin;
+	struct mcx_cmd_create_sq_out *out;
+	int error;
+	uint64_t *pas;
+	uint8_t *doorbell;
+	int insize, npages, paslen, i, token;
+
+	npages = howmany((1 << MCX_LOG_SQ_SIZE) * sizeof(struct mcx_sq_entry), PAGE_SIZE);
+	paslen = npages * sizeof(*pas);
+	insize = sizeof(struct mcx_sq_ctx) + paslen;
+
+	if (mcx_dmamem_alloc(sc, &sc->sc_sq_mem, npages * MCX_PAGE_SIZE,
+	    MCX_PAGE_SIZE) != 0) {
+		printf("%s: unable to allocate send queue memory\n", DEVNAME(sc));
+		return (-1);
+	}
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	token = mcx_cmdq_token(sc);
+	mcx_cmdq_init(sc, cqe, sizeof(*in) + insize + paslen, sizeof(*out), token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_CREATE_SQ);
+	in->cmd_op_mod = htobe16(0);
+
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
+	    &cqe->cq_input_ptr, token) != 0) {
+		printf("%s: unable to allocate create sq mailboxen\n", DEVNAME(sc));
+		error = -1;
+		goto free;
+	}
+	mbin = (struct mcx_sq_ctx *)(((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10);
+	mbin->sq_flags = htobe32(MCX_SQ_CTX_RLKEY |
+	    (1 << MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT));
+	mbin->sq_cqn = htobe32(cqn);
+	mbin->sq_tis_lst_sz = htobe32(1 << MCX_SQ_CTX_TIS_LST_SZ_SHIFT);
+	mbin->sq_tis_num = htobe32(sc->sc_tisn);
+	mbin->sq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC;
+	mbin->sq_wq.wq_pd = htobe32(sc->sc_pd);
+	mbin->sq_wq.wq_uar_page = htobe32(sc->sc_uar);
+	mbin->sq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) +
+	    MCX_SQ_DOORBELL_OFFSET);
+	mbin->sq_wq.wq_log_stride = htobe16(6);	/* 6?  maybe we should use 7 to allow lots of segments */
+	mbin->sq_wq.wq_log_size = MCX_LOG_SQ_SIZE;
+
+	/* physical addresses follow the mailbox in data */
+	pas = (uint64_t *)(mbin + 1);
+	for (i = 0; i < npages; i++) {
+		pas[i] = htobe64(MCX_DMA_DVA(&sc->sc_sq_mem) + (i * MCX_PAGE_SIZE));
+	}
+	mcx_cmdq_post(sc, cqe, 0);
+
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf("%s: create sq timeout\n", DEVNAME(sc));
+		goto free;
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf("%s: create sq command corrupt\n", DEVNAME(sc));
+		goto free;
+	}
+
+	out = mcx_cmdq_out(cqe);
+	if (out->cmd_status != MCX_CQ_STATUS_OK) {
+		printf("%s: create sq failed (%x, %x)\n", DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
+		error = -1;
+		goto free;
+	}
+
+	sc->sc_sqn = betoh32(out->cmd_sqn);
+
+	doorbell = MCX_DMA_KVA(&sc->sc_doorbell_mem);
+	sc->sc_tx_doorbell = (uint32_t *)(doorbell + MCX_SQ_DOORBELL_OFFSET + 4);
+free:
+	mcx_dmamem_free(sc, &mxm);
+	return (error);
+}
+
+static int
+mcx_ready_sq(struct mcx_softc *sc)
+{
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_dmamem mxm;
+	struct mcx_cmd_modify_sq_in *in;
+	struct mcx_cmd_modify_sq_mb_in *mbin;
+	struct mcx_cmd_modify_sq_out *out;
+	int error;
+	int token;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	token = mcx_cmdq_token(sc);
+	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_MODIFY_SQ);
+	in->cmd_op_mod = htobe16(0);
+	in->cmd_sq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | sc->sc_sqn);
+
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) {
+		printf("%s: unable to allocate modify sq mailbox\n", DEVNAME(sc));
+		return (-1);
+	}
+	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
+	mbin->cmd_sq_ctx.sq_flags = htobe32(MCX_QUEUE_STATE_RDY << MCX_SQ_CTX_STATE_SHIFT);
+
+	mcx_cmdq_mboxes_sign(&mxm, 1);
+	mcx_cmdq_post(sc, cqe, 0);
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf("%s: modify sq timeout\n", DEVNAME(sc));
+		goto free;
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf("%s: modify sq command corrupt\n", DEVNAME(sc));
+		goto free;
+	}
+
+	out = mcx_cmdq_out(cqe);
+	if (out->cmd_status != MCX_CQ_STATUS_OK) {
+		printf("%s: modify sq failed (%x, %x)\n", DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
+		error = -1;
+		goto free;
+	}
+
+free:
+	mcx_dmamem_free(sc, &mxm);
+	return (error);
+}
+
+static int
+mcx_create_tis(struct mcx_softc *sc)
+{
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_dmamem mxm;
+	struct mcx_cmd_create_tis_in *in;
+	struct mcx_cmd_create_tis_mb_in *mbin;
+	struct mcx_cmd_create_tis_out *out;
+	int error;
+	int token;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	token = mcx_cmdq_token(sc);
+	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIS);
+	in->cmd_op_mod = htobe16(0);
+
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) {
+		printf("%s: unable to allocate create tis mailbox\n", DEVNAME(sc));
+		return (-1);
+	}
+	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
+	mbin->cmd_tdomain = htobe32(sc->sc_tdomain);
+
+	mcx_cmdq_mboxes_sign(&mxm, 1);
+	mcx_cmdq_post(sc, cqe, 0);
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf("%s: create tis timeout\n", DEVNAME(sc));
+		goto free;
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf("%s: create tis command corrupt\n", DEVNAME(sc));
+		goto free;
+	}
+
+	out = mcx_cmdq_out(cqe);
+	if (out->cmd_status != MCX_CQ_STATUS_OK) {
+		printf("%s: create tis failed (%x, %x)\n", DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
+		error = -1;
+		goto free;
+	}
+
+	sc->sc_tisn = betoh32(out->cmd_tisn);
+free:
+	mcx_dmamem_free(sc, &mxm);
+	return (error);
+}
+
+static int
+mcx_alloc_flow_counter(struct mcx_softc *sc, int i)
+{
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_cmd_alloc_flow_counter_in *in;
+	struct mcx_cmd_alloc_flow_counter_out *out;
+	int error;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_ALLOC_FLOW_COUNTER);
+	in->cmd_op_mod = htobe16(0);
+
+	mcx_cmdq_post(sc, cqe, 0);
+
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf("%s: alloc flow counter timeout\n", DEVNAME(sc));
+		return (-1);
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf("%s: alloc flow counter command corrupt\n", DEVNAME(sc));
+		return (-1);
+	}
+
+	out = (struct mcx_cmd_alloc_flow_counter_out *)cqe->cq_output_data;
+	if (out->cmd_status != MCX_CQ_STATUS_OK) {
+		printf("%s: alloc flow counter failed (%x)\n", DEVNAME(sc), out->cmd_status);
+		return (-1);
+	}
+
+	sc->sc_flow_counter_id[i]  = betoh16(out->cmd_flow_counter_id);
+	printf("flow counter id %d = %d\n", i, sc->sc_flow_counter_id[i]);
+
+	return (0);
+}
+
+static int
+mcx_create_flow_table(struct mcx_softc *sc)
+{
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_dmamem mxm;
+	struct mcx_cmd_create_flow_table_in *in;
+	struct mcx_cmd_create_flow_table_mb_in *mbin;
+	struct mcx_cmd_create_flow_table_out *out;
+	int error;
+	int token;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	token = mcx_cmdq_token(sc);
+	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_TABLE);
+	in->cmd_op_mod = htobe16(0);
+
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) {
+		printf("%s: unable to allocate create flow table mailbox\n", DEVNAME(sc));
+		return (-1);
+	}
+	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
+	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
+	mbin->cmd_ctx.ft_log_size = 4;		/* how big should this be? */
+
+	mcx_cmdq_mboxes_sign(&mxm, 1);
+	mcx_cmdq_post(sc, cqe, 0);
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf("%s: create flow table timeout\n", DEVNAME(sc));
+		goto free;
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf("%s: create flow table command corrupt\n", DEVNAME(sc));
+		goto free;
+	}
+
+	out = mcx_cmdq_out(cqe);
+	if (out->cmd_status != MCX_CQ_STATUS_OK) {
+		printf("%s: create flow table failed (%x, %x)\n", DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
+		error = -1;
+		goto free;
+	}
+
+	sc->sc_flow_table_id = betoh32(out->cmd_table_id);
+free:
+	mcx_dmamem_free(sc, &mxm);
+	return (error);
+}
+
+static int
+mcx_set_flow_table_root(struct mcx_softc *sc)
+{
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_dmamem mxm;
+	struct mcx_cmd_set_flow_table_root_in *in;
+	struct mcx_cmd_set_flow_table_root_mb_in *mbin;
+	struct mcx_cmd_set_flow_table_root_out *out;
+	int error;
+	int token;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	token = mcx_cmdq_token(sc);
+	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ROOT);
+	in->cmd_op_mod = htobe16(0);
+
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) {
+		printf("%s: unable to allocate set flow table root mailbox\n", DEVNAME(sc));
+		return (-1);
+	}
+	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
+	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
+	mbin->cmd_table_id = htobe32(sc->sc_flow_table_id);
+
+	mcx_cmdq_mboxes_sign(&mxm, 1);
+	mcx_cmdq_post(sc, cqe, 0);
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf("%s: set flow table root timeout\n", DEVNAME(sc));
+		goto free;
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf("%s: set flow table root command corrupt\n", DEVNAME(sc));
+		goto free;
+	}
+
+	out = mcx_cmdq_out(cqe);
+	if (out->cmd_status != MCX_CQ_STATUS_OK) {
+		printf("%s: set flow table root failed (%x, %x)\n", DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
+		error = -1;
+		goto free;
+	}
+
+free:
+	mcx_dmamem_free(sc, &mxm);
+	return (error);
+}
+
+static int
+mcx_create_flow_group(struct mcx_softc *sc)
+{
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_dmamem mxm;
+	struct mcx_cmd_create_flow_group_in *in;
+	struct mcx_cmd_create_flow_group_mb_in *mbin;
+	struct mcx_cmd_create_flow_group_out *out;
+	int error;
+	int token;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	token = mcx_cmdq_token(sc);
+	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_GROUP);
+	in->cmd_op_mod = htobe16(0);
+
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) != 0) {
+		printf("%s: unable to allocate create flow group mailbox\n", DEVNAME(sc));
+		return (-1);
+	}
+	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
+	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
+	mbin->cmd_table_id = htobe32(sc->sc_flow_table_id);
+	/* no match criteria, include flow index 0-2 */
+	mbin->cmd_start_flow_index = 0;
+	mbin->cmd_end_flow_index = htobe32(3);
+	mbin->cmd_match_criteria_enable = 0;
+
+	mcx_cmdq_mboxes_sign(&mxm, 2);
+	mcx_cmdq_post(sc, cqe, 0);
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf("%s: create flow group timeout\n", DEVNAME(sc));
+		goto free;
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf("%s: create flow group command corrupt\n", DEVNAME(sc));
+		goto free;
+	}
+
+	out = mcx_cmdq_out(cqe);
+	if (out->cmd_status != MCX_CQ_STATUS_OK) {
+		printf("%s: create flow group failed (%x, %x)\n", DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
+		error = -1;
+		goto free;
+	}
+
+	sc->sc_flow_group_id = betoh32(out->cmd_group_id);
+
+free:
+	mcx_dmamem_free(sc, &mxm);
+	return (error);
+}
+
+static int
+mcx_set_flow_table_entry(struct mcx_softc *sc, int counter)
+{
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_dmamem mxm;
+	struct mcx_cmd_set_flow_table_entry_in *in;
+	struct mcx_cmd_set_flow_table_entry_mb_in *mbin;
+	struct mcx_cmd_set_flow_table_entry_out *out;
+	uint32_t *dest;
+	int error;
+	int token;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	token = mcx_cmdq_token(sc);
+	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*dest), sizeof(*out), token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY);
+	in->cmd_op_mod = htobe16(0);
+
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) != 0) {
+		printf("%s: unable to allocate set flow table entry mailbox\n", DEVNAME(sc));
+		return (-1);
+	}
+	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
+	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
+	mbin->cmd_table_id = htobe32(sc->sc_flow_table_id);
+	mbin->cmd_flow_index = htobe32(sc->sc_flow_table_entry);
+	mbin->cmd_flow_ctx.fc_group_id = htobe32(sc->sc_flow_group_id);
+
+	/* flow context ends at offset 0x330, which is 0x130 into the second mbox */
+	dest = (uint32_t *)(((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130);
+	switch (counter) {
+	case -3:
+		mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_ALLOW);
+		break;
+	case -2:
+		mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_DROP);
+		break;
+
+	case -1:
+		mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD);
+		mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1);
+		*dest = htobe32(sc->sc_tirn | MCX_FLOW_CONTEXT_DEST_TYPE_TIR);
+		break;
+
+	default:
+		mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_ALLOW |
+		    MCX_FLOW_CONTEXT_ACTION_COUNT);
+		mbin->cmd_flow_ctx.fc_counter_list_size = htobe32(1);
+		*dest = htobe32(sc->sc_flow_counter_id[counter]);
+	}
+	sc->sc_flow_table_entry++;
+
+	mcx_cmdq_mboxes_sign(&mxm, 2);
+	mcx_cmdq_post(sc, cqe, 0);
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf("%s: set flow table entry timeout\n", DEVNAME(sc));
+		goto free;
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf("%s: set flow table entry command corrupt\n", DEVNAME(sc));
+		goto free;
+	}
+
+	out = mcx_cmdq_out(cqe);
+	if (out->cmd_status != MCX_CQ_STATUS_OK) {
+		printf("%s: set flow table entry failed (%x, %x)\n", DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
+		error = -1;
+		goto free;
+	}
+
+free:
+	mcx_dmamem_free(sc, &mxm);
+	return (error);
+}
+
+#if 0
+int
+mcx_dump_flow_table(struct mcx_softc *sc)
+{
+	struct mcx_dmamem mxm;
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_cmd_query_flow_table_in *in;
+	struct mcx_cmd_query_flow_table_mb_in *mbin;
+	struct mcx_cmd_query_flow_table_out *out;
+	struct mcx_cmd_query_flow_table_mb_out *mbout;
+	uint8_t token = mcx_cmdq_token(sc);
+	int error;
+	int i;
+	uint8_t *dump;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out) + sizeof(*mbout) + 16, token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE);
+	in->cmd_op_mod = htobe16(0);
+
+	CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
+	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE);
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
+	    &cqe->cq_output_ptr, token) != 0) {
+		printf(", unable to allocate query flow table mailboxes\n");
+		return (-1);
+	}
+	cqe->cq_input_ptr = cqe->cq_output_ptr;
+
+	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
+	mbin->cmd_table_type = 0;
+	mbin->cmd_table_id = htobe32(sc->sc_flow_table_id);
+
+	mcx_cmdq_mboxes_sign(&mxm, 1);
+
+	mcx_cmdq_post(sc, cqe, 0);
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf("%s: query flow table timeout\n", DEVNAME(sc));
+		goto free;
+	}
+	error = mcx_cmdq_verify(cqe);
+	if (error != 0) {
+		printf("%s: query flow table reply corrupt\n", DEVNAME(sc));
+		goto free;
+	}
+
+	out = mcx_cmdq_out(cqe);
+	switch (out->cmd_status) {
+	case MCX_CQ_STATUS_OK:
+		break;
+	default:
+		printf("%s: query flow table failed (%x/%x)\n", DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
+		error = -1;
+		goto free;
+	}
+
+        mbout = (struct mcx_cmd_query_flow_table_mb_out *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
+	dump = (uint8_t *)mbout + 8;
+	for (i = 0; i < sizeof(struct mcx_flow_table_ctx); i++) {
+		printf("%.2x ", dump[i]);
+		if (i % 16 == 15)
+			printf("\n");
+	}
+free:
+	mcx_cq_mboxes_free(sc, &mxm);
+	return (error);
+}
+int
+mcx_dump_flow_table_entry(struct mcx_softc *sc, int index)
+{
+	struct mcx_dmamem mxm;
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_cmd_query_flow_table_entry_in *in;
+	struct mcx_cmd_query_flow_table_entry_mb_in *mbin;
+	struct mcx_cmd_query_flow_table_entry_out *out;
+	struct mcx_cmd_query_flow_table_entry_mb_out *mbout;
+	uint8_t token = mcx_cmdq_token(sc);
+	int error;
+	int i;
+	uint8_t *dump;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out) + sizeof(*mbout) + 16, token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE_ENTRY);
+	in->cmd_op_mod = htobe16(0);
+
+	CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
+	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
+	    &cqe->cq_output_ptr, token) != 0) {
+		printf(", unable to allocate query flow table entry mailboxes\n");
+		return (-1);
+	}
+	cqe->cq_input_ptr = cqe->cq_output_ptr;
+
+	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
+	mbin->cmd_table_type = 0;
+	mbin->cmd_table_id = htobe32(sc->sc_flow_table_id);
+	mbin->cmd_flow_index = htobe32(index);
+
+	mcx_cmdq_mboxes_sign(&mxm, 1);
+
+	mcx_cmdq_post(sc, cqe, 0);
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf("%s: query flow table entry timeout\n", DEVNAME(sc));
+		goto free;
+	}
+	error = mcx_cmdq_verify(cqe);
+	if (error != 0) {
+		printf("%s: query flow table entry reply corrupt\n", DEVNAME(sc));
+		goto free;
+	}
+
+	out = mcx_cmdq_out(cqe);
+	switch (out->cmd_status) {
+	case MCX_CQ_STATUS_OK:
+		break;
+	default:
+		printf("%s: query flow table entry failed (%x/%x)\n", DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
+		error = -1;
+		goto free;
+	}
+
+        mbout = (struct mcx_cmd_query_flow_table_entry_mb_out *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
+	dump = (uint8_t *)mbout;
+	for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
+		printf("%.2x ", dump[i]);
+		if (i % 16 == 15)
+			printf("\n");
+	}
+
+free:
+	mcx_cq_mboxes_free(sc, &mxm);
+	return (error);
+}
+
+int
+mcx_dump_flow_group(struct mcx_softc *sc)
+{
+	struct mcx_dmamem mxm;
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_cmd_query_flow_group_in *in;
+	struct mcx_cmd_query_flow_group_mb_in *mbin;
+	struct mcx_cmd_query_flow_group_out *out;
+	struct mcx_cmd_query_flow_group_mb_out *mbout;
+	uint8_t token = mcx_cmdq_token(sc);
+	int error;
+	int i;
+	uint8_t *dump;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out) + sizeof(*mbout) + 16, token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_GROUP);
+	in->cmd_op_mod = htobe16(0);
+
+	CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
+	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
+	    &cqe->cq_output_ptr, token) != 0) {
+		printf(", unable to allocate query flow group mailboxes\n");
+		return (-1);
+	}
+	cqe->cq_input_ptr = cqe->cq_output_ptr;
+
+	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
+	mbin->cmd_table_type = 0;
+	mbin->cmd_table_id = htobe32(sc->sc_flow_table_id);
+	mbin->cmd_group_id = htobe32(sc->sc_flow_group_id);
+
+	mcx_cmdq_mboxes_sign(&mxm, 1);
+
+	mcx_cmdq_post(sc, cqe, 0);
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf("%s: query flow group timeout\n", DEVNAME(sc));
+		goto free;
+	}
+	error = mcx_cmdq_verify(cqe);
+	if (error != 0) {
+		printf("%s: query flow group reply corrupt\n", DEVNAME(sc));
+		goto free;
+	}
+
+	out = mcx_cmdq_out(cqe);
+	switch (out->cmd_status) {
+	case MCX_CQ_STATUS_OK:
+		break;
+	default:
+		printf("%s: query flow group failed (%x/%x)\n", DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
+		error = -1;
+		goto free;
+	}
+
+        mbout = (struct mcx_cmd_query_flow_group_mb_out *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
+	dump = (uint8_t *)mbout;
+	for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
+		printf("%.2x ", dump[i]);
+		if (i % 16 == 15)
+			printf("\n");
+	}
+	dump = (uint8_t *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1)));
+	for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
+		printf("%.2x ", dump[i]);
+		if (i % 16 == 15)
+			printf("\n");
+	}
+
+free:
+	mcx_cq_mboxes_free(sc, &mxm);
+	return (error);
+}
+
+int
+mcx_dump_rq(struct mcx_softc *sc)
+{
+	struct mcx_dmamem mxm;
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_cmd_query_rq_in *in;
+	struct mcx_cmd_query_rq_out *out;
+	struct mcx_cmd_query_rq_mb_out *mbout;
+	uint8_t token = mcx_cmdq_token(sc);
+	int error;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16, token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_QUERY_RQ);
+	in->cmd_op_mod = htobe16(0);
+	in->cmd_rqn = htobe32(sc->sc_rqn);
+
+	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
+	    &cqe->cq_output_ptr, token) != 0) {
+		printf(", unable to allocate query flow group mailboxes\n");
+		return (-1);
+	}
+
+	mcx_cmdq_mboxes_sign(&mxm, 1);
+
+	mcx_cmdq_post(sc, cqe, 0);
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf("%s: query rq timeout\n", DEVNAME(sc));
+		goto free;
+	}
+	error = mcx_cmdq_verify(cqe);
+	if (error != 0) {
+		printf("%s: query rq reply corrupt\n", DEVNAME(sc));
+		goto free;
+	}
+
+	out = mcx_cmdq_out(cqe);
+	switch (out->cmd_status) {
+	case MCX_CQ_STATUS_OK:
+		break;
+	default:
+		printf("%s: query rq failed (%x/%x)\n", DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
+		error = -1;
+		goto free;
+	}
+
+        mbout = (struct mcx_cmd_query_rq_mb_out *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
+	printf("%s: rq: state %d, ui %d, cqn %d, s/s %d/%d/%d, hw %d, sw %d\n", DEVNAME(sc),
+	    (betoh32(mbout->cmd_ctx.rq_flags) >> MCX_RQ_CTX_STATE_SHIFT) & 0x0f,
+	    betoh32(mbout->cmd_ctx.rq_user_index),
+	    betoh32(mbout->cmd_ctx.rq_cqn),
+	    betoh16(mbout->cmd_ctx.rq_wq.wq_log_stride), mbout->cmd_ctx.rq_wq.wq_log_page_sz,
+	    mbout->cmd_ctx.rq_wq.wq_log_size,
+	    betoh32(mbout->cmd_ctx.rq_wq.wq_hw_counter),
+	    betoh32(mbout->cmd_ctx.rq_wq.wq_sw_counter));
+
+free:
+	mcx_cq_mboxes_free(sc, &mxm);
+	return (error);
+}
+
+int
+mcx_dump_sq(struct mcx_softc *sc)
+{
+	struct mcx_dmamem mxm;
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_cmd_query_sq_in *in;
+	struct mcx_cmd_query_sq_out *out;
+	struct mcx_cmd_query_sq_mb_out *mbout;
+	uint8_t token = mcx_cmdq_token(sc);
+	int error;
+	int i;
+	uint8_t *dump;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16, token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_QUERY_SQ);
+	in->cmd_op_mod = htobe16(0);
+	in->cmd_sqn = htobe32(sc->sc_sqn);
+
+	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
+	    &cqe->cq_output_ptr, token) != 0) {
+		printf(", unable to allocate query sq mailboxes\n");
+		return (-1);
+	}
+
+	mcx_cmdq_mboxes_sign(&mxm, 1);
+
+	mcx_cmdq_post(sc, cqe, 0);
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf("%s: query sq timeout\n", DEVNAME(sc));
+		goto free;
+	}
+	error = mcx_cmdq_verify(cqe);
+	if (error != 0) {
+		printf("%s: query sq reply corrupt\n", DEVNAME(sc));
+		goto free;
+	}
+
+	out = mcx_cmdq_out(cqe);
+	switch (out->cmd_status) {
+	case MCX_CQ_STATUS_OK:
+		break;
+	default:
+		printf("%s: query sq failed (%x/%x)\n", DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
+		error = -1;
+		goto free;
+	}
+
+        mbout = (struct mcx_cmd_query_sq_mb_out *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
+/*
+	printf("%s: rq: state %d, ui %d, cqn %d, s/s %d/%d/%d, hw %d, sw %d\n", DEVNAME(sc),
+	    (betoh32(mbout->cmd_ctx.rq_flags) >> MCX_RQ_CTX_STATE_SHIFT) & 0x0f,
+	    betoh32(mbout->cmd_ctx.rq_user_index),
+	    betoh32(mbout->cmd_ctx.rq_cqn),
+	    betoh16(mbout->cmd_ctx.rq_wq.wq_log_stride), mbout->cmd_ctx.rq_wq.wq_log_page_sz,
+	    mbout->cmd_ctx.rq_wq.wq_log_size,
+	    betoh32(mbout->cmd_ctx.rq_wq.wq_hw_counter),
+	    betoh32(mbout->cmd_ctx.rq_wq.wq_sw_counter));
+*/
+	dump = (uint8_t *)mbout;
+	for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
+		printf("%.2x ", dump[i]);
+		if (i % 16 == 15)
+			printf("\n");
+	}
+
+free:
+	mcx_cq_mboxes_free(sc, &mxm);
+	return (error);
+}
+
+static int
+mcx_dump_counters(struct mcx_softc *sc)
+{
+	struct mcx_dmamem mxm;
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_cmd_query_vport_counters_in *in;
+	struct mcx_cmd_query_vport_counters_mb_in *mbin;
+	struct mcx_cmd_query_vport_counters_out *out;
+	struct mcx_nic_vport_counters *counters;
+	int error, token;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	token = mcx_cmdq_token(sc);
+	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out) + sizeof(*counters), token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_QUERY_VPORT_COUNTERS);
+	in->cmd_op_mod = htobe16(0);
+
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_output_ptr, token) != 0) {
+		printf(", unable to allocate query nic vport counters mailboxen\n");
+		return (-1);
+	}
+	cqe->cq_input_ptr = cqe->cq_output_ptr;
+
+	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
+	mbin->cmd_clear = 0x80;
+
+	mcx_cmdq_mboxes_sign(&mxm, 1);
+	mcx_cmdq_post(sc, cqe, 0);
+
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf("%s: query nic vport counters timeout\n", DEVNAME(sc));
+		goto free;
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf("%s: query nic vport counters command corrupt\n", DEVNAME(sc));
+		goto free;
+	}
+
+	out = mcx_cmdq_out(cqe);
+	if (out->cmd_status != MCX_CQ_STATUS_OK) {
+		printf("%s: query nic vport counters failed (%x, %x)\n", DEVNAME(sc), out->cmd_status, out->cmd_syndrome);
+		error = -1;
+		goto free;
+	}
+
+	counters = (struct mcx_nic_vport_counters *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
+	if (counters->rx_bcast.packets + counters->tx_bcast.packets + counters->rx_ucast.packets + counters->tx_ucast.packets + counters->rx_err.packets + counters->tx_err.packets)
+		printf("%s: err %llx/%llx uc %llx/%llx bc %llx/%llx\n", DEVNAME(sc),
+		    betoh64(counters->tx_err.packets), betoh64(counters->rx_err.packets),
+		    betoh64(counters->tx_ucast.packets), betoh64(counters->rx_ucast.packets),
+		    betoh64(counters->tx_bcast.packets), betoh64(counters->rx_bcast.packets));
+free:
+	mcx_dmamem_free(sc, &mxm);
+
+	return (error);
+}
+
+static int
+mcx_dump_flow_counter(struct mcx_softc *sc, int index, const char *what)
+{
+	struct mcx_dmamem mxm;
+	struct mcx_cmdq_entry *cqe;
+	struct mcx_cmd_query_flow_counter_in *in;
+	struct mcx_cmd_query_flow_counter_mb_in *mbin;
+	struct mcx_cmd_query_flow_counter_out *out;
+	struct mcx_counter *counters;
+	int error, token;
+
+	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
+	token = mcx_cmdq_token(sc);
+	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out) + sizeof(*counters), token);
+
+	in = mcx_cmdq_in(cqe);
+	in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_COUNTER);
+	in->cmd_op_mod = htobe16(0);
+
+	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_output_ptr, token) != 0) {
+		printf(", unable to allocate query flow counter mailboxen\n");
+		return (-1);
+	}
+	cqe->cq_input_ptr = cqe->cq_output_ptr;
+	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
+	mbin->cmd_flow_counter_id = htobe16(sc->sc_flow_counter_id[index]);
+	mbin->cmd_clear = 0x80;
+
+	mcx_cmdq_mboxes_sign(&mxm, 1);
+	mcx_cmdq_post(sc, cqe, 0);
+
+	error = mcx_cmdq_poll(sc, cqe, 1000);
+	if (error != 0) {
+		printf("%s: query flow counter timeout\n", DEVNAME(sc));
+		goto free;
+	}
+	if (mcx_cmdq_verify(cqe) != 0) {
+		printf("%s: query flow counter command corrupt\n", DEVNAME(sc));
+		goto free;
+	}
+
+	out = mcx_cmdq_out(cqe);
+	if (out->cmd_status != MCX_CQ_STATUS_OK) {
+		printf("%s: query flow counter failed (%x, %x)\n", DEVNAME(sc), out->cmd_status, out->cmd_syndrome);
+		error = -1;
+		goto free;
+	}
+
+	counters = (struct mcx_counter *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
+	if (counters->packets)
+		printf("%s: %s inflow %llx\n", DEVNAME(sc), what, betoh64(counters->packets));
+free:
+	mcx_dmamem_free(sc, &mxm);
+
+	return (error);
+}
+
+#endif
+
+int
+mcx_rx_fill_slots(struct mcx_softc *sc, void *ring, struct mcx_slot *slots, uint *prod, int bufsize, uint nslots)
+{
+	struct mcx_rq_entry *rqe;
+	struct mcx_slot *ms;
+	struct mbuf *m;
+	uint slot, p, fills;
+
+	p = *prod;
+	slot = (p % (1 << MCX_LOG_RQ_SIZE));
+	rqe = ring;
+	for (fills = 0; fills < nslots; fills++) {
+		ms = &slots[slot];
+		m = MCLGETI(NULL, M_DONTWAIT, NULL, bufsize);
+		if (m == NULL)
+			break;
+
+		m->m_len = m->m_pkthdr.len = bufsize;
+		if (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
+		    BUS_DMA_NOWAIT) != 0) {
+			m_freem(m);
+			break;
+		}
+		ms->ms_m = m;
+
+		rqe[slot].rqe_byte_count = htobe32(bufsize);
+		rqe[slot].rqe_addr = htobe64(ms->ms_map->dm_segs[0].ds_addr);
+		rqe[slot].rqe_lkey = htobe32(sc->sc_lkey);
+
+		p++;
+		slot++;
+		if (slot == (1 << MCX_LOG_RQ_SIZE))
+			slot = 0;
+	}
+
+	if (fills != 0) {
+		*sc->sc_rx_doorbell = htobe32(p);
+		/* barrier? */
+	}
+
+	*prod = p;
+
+	return (nslots - fills);
+}
+
+int
+mcx_rx_fill(struct mcx_softc *sc)
+{
+	u_int slots;
+
+	slots = if_rxr_get(&sc->sc_rxr, (1 << MCX_LOG_RQ_SIZE));
+	if (slots == 0)
+		return (1);
+
+	slots = mcx_rx_fill_slots(sc, MCX_DMA_KVA(&sc->sc_rq_mem), sc->sc_rx_slots,
+	    &sc->sc_rx_prod, MCLBYTES, slots);
+	if_rxr_put(&sc->sc_rxr, slots);
+	return (0);
+}
+
+void
+mcx_refill(void *xsc)
+{
+	struct mcx_softc *sc = xsc;
+
+	mcx_rx_fill(sc);
+
+	if (sc->sc_rx_cons == sc->sc_rx_prod)
+		timeout_add(&sc->sc_rx_refill, 1);
+}
+
+void
+mcx_process_txeof(struct mcx_softc *sc, struct mcx_cq_entry *cqe, int *txfree)
+{
+	struct mcx_slot *ms;
+	bus_dmamap_t map;
+	int slot;
+
+	slot = betoh16(cqe->cq_wqe_count) % (1 << MCX_LOG_SQ_SIZE);
+
+	ms = &sc->sc_tx_slots[slot];
+	map = ms->ms_map;
+	bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
+	bus_dmamap_unload(sc->sc_dmat, map);
+	m_freem(ms->ms_m);
+
+	(*txfree)++;
+}
+
+void
+mcx_process_rx(struct mcx_softc *sc, struct mcx_cq_entry *cqe, struct mbuf_list *ml, int *slots)
+{
+	struct mcx_slot *ms;
+	struct mbuf *m;
+	int slot;
+
+	slot = betoh16(cqe->cq_wqe_count) % (1 << MCX_LOG_RQ_SIZE);
+
+	ms = &sc->sc_rx_slots[slot];
+	bus_dmamap_sync(sc->sc_dmat, ms->ms_map, 0, ms->ms_map->dm_mapsize,
+	    BUS_DMASYNC_POSTREAD);
+	bus_dmamap_unload(sc->sc_dmat, ms->ms_map);
+
+	m = ms->ms_m;
+	ms->ms_m = NULL;
+	m->m_pkthdr.len = m->m_len = betoh32(cqe->cq_byte_cnt);
+	(*slots)++;
+
+	ml_enqueue(ml, m);
+}
+
+static struct mcx_cq_entry *
+mcx_next_cq_entry(struct mcx_softc *sc, struct mcx_cq *cq)
+{
+	struct mcx_cq_entry *cqe;
+	int next;
+
+	cqe = (struct mcx_cq_entry *)MCX_DMA_KVA(&cq->cq_mem);
+	next = cq->cq_cons % (1 << MCX_LOG_CQ_SIZE);
+
+	if ((cqe[next].cq_opcode_owner & MCX_CQ_ENTRY_FLAG_OWNER) ==
+	    ((cq->cq_cons >> MCX_LOG_CQ_SIZE) & 1)) {
+		return (&cqe[next]);
+	}
+
+	return (NULL);
+}
+
+static void
+mcx_arm_cq(struct mcx_softc *sc, struct mcx_cq *cq)
+{
+	bus_size_t offset;
+	uint32_t val;
+	uint64_t uval;
+
+	/* different uar per cq? */
+	offset = (MCX_PAGE_SIZE * sc->sc_uar);
+	val = ((cq->cq_count) & 3) << MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT;
+	val |= (cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK);
+
+	cq->cq_doorbell[0] = htobe32(cq->cq_cons);
+	cq->cq_doorbell[1] = htobe32(val);
+
+	uval = val;
+	uval <<= 32;
+	uval |= cq->cq_n;
+	bus_space_write_raw_8(sc->sc_memt, sc->sc_memh, offset + MCX_UAR_CQ_DOORBELL,
+	    htobe64(uval));
+	/* barrier? */
+}
+
+void
+mcx_process_cq(struct mcx_softc *sc, struct mcx_cq *cq)
+{
+	struct ifnet *ifp = &sc->sc_ac.ac_if;
+	struct mcx_cq_entry *cqe;
+	uint8_t *cqp;
+	int count = 0;
+	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
+	int rxfree, txfree;
+
+	rxfree = 0;
+	txfree = 0;
+	while ((cqe = mcx_next_cq_entry(sc, cq))) {
+		uint8_t opcode;
+		opcode = (cqe->cq_opcode_owner >> MCX_CQ_ENTRY_OPCODE_SHIFT);
+		switch (opcode) {
+		case MCX_CQ_ENTRY_OPCODE_REQ:
+			mcx_process_txeof(sc, cqe, &txfree);
+			break;
+		case MCX_CQ_ENTRY_OPCODE_SEND:
+			mcx_process_rx(sc, cqe, &ml, &rxfree);
+			break;
+		case MCX_CQ_ENTRY_OPCODE_REQ_ERR:
+		case MCX_CQ_ENTRY_OPCODE_SEND_ERR:
+			cqp = (uint8_t *)cqe;
+			/* printf("%s: cq completion error: %x\n", DEVNAME(sc), cqp[0x37]); */
+			break;
+
+		default:
+			/* printf("%s: cq completion opcode %x??\n", DEVNAME(sc), opcode); */
+			break;
+		}
+
+		count++;
+		cq->cq_cons++;
+	}
+
+	if (count > 0) {
+		cq->cq_count++;
+		mcx_arm_cq(sc, cq);
+	}
+
+	if (rxfree > 0) {
+		if_rxr_put(&sc->sc_rxr, rxfree);
+		mcx_rx_fill(sc);
+		/* timeout if full somehow */
+
+		if_input(&sc->sc_ac.ac_if, &ml);
+	}
+	if (txfree > 0) {
+		sc->sc_tx_cons += txfree;
+		if (ifq_is_oactive(&ifp->if_snd))
+			ifq_restart(&ifp->if_snd);
+	}
+}
+
+static void
+mcx_arm_eq(struct mcx_softc *sc)
+{
+	bus_size_t offset;
+	uint32_t val;
+
+	offset = (MCX_PAGE_SIZE * sc->sc_uar) + MCX_UAR_EQ_DOORBELL_ARM;
+	val = (sc->sc_eqn << 24) | (sc->sc_eq_cons & 0xffffff);
+
+	mcx_wr(sc, offset, val);
+	/* barrier? */
+}
+
+static struct mcx_eq_entry *
+mcx_next_eq_entry(struct mcx_softc *sc)
+{
+	struct mcx_eq_entry *eqe;
+	int next;
+
+	eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&sc->sc_eq_mem);
+	next = sc->sc_eq_cons % (1 << MCX_LOG_EQ_SIZE);
+	if ((eqe[next].eq_owner & 1) == ((sc->sc_eq_cons >> MCX_LOG_EQ_SIZE) & 1)) {
+		sc->sc_eq_cons++;
+		return (&eqe[next]);
+	}
+	return (NULL);
+}
+
+int
+mcx_intr(void *xsc)
+{
+	struct mcx_softc *sc = (struct mcx_softc *)xsc;
+	struct mcx_eq_entry *eqe;
+	int i, cq;
+
+	while ((eqe = mcx_next_eq_entry(sc))) {
+		switch (eqe->eq_event_type) {
+		case MCX_EVENT_TYPE_COMPLETION:
+			cq = betoh32(eqe->eq_event_data[6]);
+			for (i = 0; i < sc->sc_num_cq; i++) {
+				if (sc->sc_cq[i].cq_n == cq) {
+					mcx_process_cq(sc, &sc->sc_cq[i]);
+					break;
+				}
+			}
+			break;
+
+		case MCX_EVENT_TYPE_LAST_WQE:
+			/* printf("%s: last wqe reached?\n", DEVNAME(sc)); */
+			break;
+
+		case MCX_EVENT_TYPE_CQ_ERROR:
+			/* printf("%s: cq error\n", DEVNAME(sc)); */
+			break;
+
+		case MCX_EVENT_TYPE_CMD_COMPLETION:
+			/* wakeup probably */
+			break;
+
+		case MCX_EVENT_TYPE_PORT_CHANGE:
+			/* printf("%s: port change\n", DEVNAME(sc)); */
+			break;
+
+		default:
+			/* printf("%s: something happened\n", DEVNAME(sc)); */
+			break;
+		}
+	}
+	mcx_arm_eq(sc);
+	return (1);
+}
+
+static void
+mcx_free_slots(struct mcx_softc *sc, struct mcx_slot *slots, int allocated, int total)
+{
+	struct mcx_slot *ms;
+
+	int i = allocated;
+	while (i-- > 0) {
+		ms = &slots[i];
+		bus_dmamap_destroy(sc->sc_dmat, ms->ms_map);
+	}
+	free(slots, M_DEVBUF, total * sizeof(*ms));
+}
+
+static void
+mcx_up(struct mcx_softc *sc)
+{
+	struct ifnet *ifp = &sc->sc_ac.ac_if;
+	struct mcx_slot *ms;
+	int i;
+
+	sc->sc_rx_slots = mallocarray(sizeof(*ms), (1 << MCX_LOG_RQ_SIZE),
+	    M_DEVBUF, M_WAITOK | M_ZERO);
+	if (sc->sc_rx_slots == NULL) {
+		printf("%s: failed to allocate rx slots\n", DEVNAME(sc));
+		return;
+	}
+
+	for (i = 0; i < (1 << MCX_LOG_RQ_SIZE); i++) {
+		ms = &sc->sc_rx_slots[i];
+		if (bus_dmamap_create(sc->sc_dmat, MCLBYTES, 1, MCLBYTES, 0,
+		    BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ms->ms_map) != 0) {
+			printf("%s: failed to allocate rx dma maps\n",
+			    DEVNAME(sc));
+			goto destroy_rx_slots;
+		}
+	}
+
+	sc->sc_tx_slots = mallocarray(sizeof(*ms), (1 << MCX_LOG_SQ_SIZE),
+	    M_DEVBUF, M_WAITOK | M_ZERO);
+	if (sc->sc_tx_slots == NULL) {
+		printf("%s: failed to allocate tx slots\n", DEVNAME(sc));
+		goto destroy_rx_slots;
+	}
+
+	for (i = 0; i < (1 << MCX_LOG_SQ_SIZE); i++) {
+		ms = &sc->sc_tx_slots[i];
+		if (bus_dmamap_create(sc->sc_dmat, MCLBYTES, 1, MCLBYTES, 0,
+		    BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ms->ms_map) != 0) {
+			printf("%s: failed to allocate tx dma maps\n",
+			    DEVNAME(sc));
+			goto destroy_tx_slots;
+		}
+	}
+
+	if (mcx_create_cq(sc, sc->sc_eqn) != 0)
+		goto destroy_tx_slots;
+	if (mcx_create_cq(sc, sc->sc_eqn) != 0)
+		goto destroy_tx_slots;
+
+	/* send queue */
+	if (mcx_create_tis(sc) != 0)
+		goto destroy_sq;
+
+	if (mcx_create_sq(sc, sc->sc_cq[0].cq_n) != 0)
+		goto destroy_rq;
+
+	/* receive queue */
+	if (mcx_create_rq(sc, sc->sc_cq[1].cq_n) != 0)
+		goto destroy_cq;
+
+	if (mcx_create_tir(sc) != 0)
+		goto destroy_tis;
+
+	if (0) {
+		if (mcx_alloc_flow_counter(sc, 0) != 0)
+			goto destroy_tir;
+		if (mcx_alloc_flow_counter(sc, 1) != 0)
+			goto destroy_tir;
+		if (mcx_alloc_flow_counter(sc, 2) != 0)
+			goto destroy_tir;
+	}
+
+	/* receive flow table mapping everything to the rq */
+	if (mcx_create_flow_table(sc) != 0)
+		goto destroy_tir;
+
+	if (mcx_create_flow_group(sc) != 0)
+		goto destroy_flow_table;	/* probably don't have to unset the root table? */
+
+	if (mcx_set_flow_table_entry(sc, -1) != 0)
+		goto destroy_flow_group;
+
+	if (mcx_set_flow_table_root(sc) != 0)
+		goto destroy_flow_table;
+
+	/* start the queues */
+	if (mcx_ready_sq(sc) != 0)
+		goto destroy_flow_table_entry;
+
+	if (mcx_ready_rq(sc) != 0)
+		goto destroy_flow_table_entry;
+
+	if_rxr_init(&sc->sc_rxr, 1, (1 << MCX_LOG_RQ_SIZE));
+	sc->sc_rx_cons = 0;
+	sc->sc_rx_prod = 0;
+	mcx_rx_fill(sc);
+
+	SET(ifp->if_flags, IFF_RUNNING);
+
+	sc->sc_tx_cons = 0;
+	sc->sc_tx_prod = 0;
+	ifq_clr_oactive(&ifp->if_snd);
+	ifq_restart(&ifp->if_snd);
+
+	return;
+destroy_flow_table_entry:
+	/* mcx_destroy_flow_table_entry(sc); */
+destroy_flow_group:
+	/* mcx_destroy_flow_group(sc); */
+destroy_flow_table:
+	/* mcx_destroy_flow_table(sc); */
+destroy_tir:
+	/* mcx_destroy_tir(sc); */
+destroy_tis:
+	/* mcx_destroy_tis(sc); */
+destroy_sq:
+	/* mcx_destroy_sq(sc); */
+destroy_rq:
+	/* mcx_destroy_rq(sc); */
+destroy_cq:
+	/* mcx_destroy_cq(sc); */
+destroy_tx_slots:
+	mcx_free_slots(sc, sc->sc_tx_slots, i, (1 << MCX_LOG_SQ_SIZE));
+	sc->sc_rx_slots = NULL;
+
+	i = (1 << MCX_LOG_RQ_SIZE);
+destroy_rx_slots:
+	mcx_free_slots(sc, sc->sc_rx_slots, i, (1 << MCX_LOG_RQ_SIZE));
+	sc->sc_rx_slots = NULL;
+}
+
+static void
+mcx_down(struct mcx_softc *sc)
+{
+	/* destroy all the rings and stuff? */
+}
+
+static int
+mcx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
+{
+	struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
+	struct ifreq *ifr = (struct ifreq *)data;
+	int s, error = 0;
+
+	s = splnet();
+	switch (cmd) {
+	case SIOCSIFADDR:
+		ifp->if_flags |= IFF_UP;
+		/* FALLTHROUGH */
+
+	case SIOCSIFFLAGS:
+		if (ISSET(ifp->if_flags, IFF_UP)) {
+			if (ISSET(ifp->if_flags, IFF_RUNNING))
+				error = ENETRESET;
+			else
+				mcx_up(sc);
+		} else {
+			if (ISSET(ifp->if_flags, IFF_RUNNING))
+				mcx_down(sc);
+		}
+		break;
+
+	case SIOCGIFMEDIA:
+	case SIOCSIFMEDIA:
+		error = ifmedia_ioctl(ifp, ifr, &sc->sc_media, cmd);
+		break;
+
+	case SIOCGIFRXR:
+		error = mcx_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data);
+		break;
+
+	default:
+		error = ether_ioctl(ifp, &sc->sc_ac, cmd, data);
+	}
+
+	if (error == ENETRESET) {
+		if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) ==
+		    (IFF_UP | IFF_RUNNING))
+			mcx_iff(sc);
+		error = 0;
+	}
+	splx(s);
+
+	return (error);
+}
+
+static int
+mcx_rxrinfo(struct mcx_softc *sc, struct if_rxrinfo *ifri)
+{
+	struct if_rxring_info ifr;
+
+	memset(&ifr, 0, sizeof(ifr));
+	ifr.ifr_size = MCLBYTES;
+	ifr.ifr_info = sc->sc_rxr;
+
+	return (if_rxr_info_ioctl(ifri, 1, &ifr));
+}
+
+static void
+mcx_start(struct ifqueue *ifq)
+{
+	struct ifnet *ifp = ifq->ifq_if;
+	struct mcx_softc *sc = ifp->if_softc;
+	struct mcx_sq_entry *sq, *sqe;
+	struct mcx_slot *ms;
+	bus_dmamap_t map;
+	struct mbuf *m;
+	u_int idx, free, used;
+	uint64_t *bf;
+	size_t bf_base;
+
+	bf_base = (sc->sc_uar * MCX_PAGE_SIZE) + MCX_UAR_BF;
+
+	idx = sc->sc_tx_prod % (1 << MCX_LOG_SQ_SIZE);
+	free = (sc->sc_tx_cons + (1 << MCX_LOG_SQ_SIZE)) - sc->sc_tx_prod;
+
+	used = 0;
+	sq = (struct mcx_sq_entry *)MCX_DMA_KVA(&sc->sc_sq_mem);
+
+	for (;;) {
+		/* each packet takes exactly one sqe */
+		if (used == free) {
+			printf("out of space\n");
+			ifq_set_oactive(ifq);
+			break;
+		}
+
+		m = ifq_dequeue(ifq);
+		if (m == NULL) {
+			break;
+		}
+
+		sqe = sq + idx;
+		ms = &sc->sc_tx_slots[idx];
+
+		/* m_defrag on single buf chains is a no op */
+		if (m_defrag(m, M_DONTWAIT)) {
+			printf("failed to defrag\n");
+			m_freem(m);
+			ifp->if_oerrors++;
+			continue;
+		}
+		if (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
+		    BUS_DMA_STREAMING | BUS_DMA_NOWAIT)) {
+			printf("failed to load mbuf\n");
+			m_freem(m);
+			ifp->if_oerrors++;
+			continue;
+		}
+
+		ms->ms_m = m;
+
+#if NBPFILTER > 0
+		if (ifp->if_bpf)
+			bpf_mtap(ifp->if_bpf, m, BPF_DIRECTION_OUT);
+#endif
+		map = ms->ms_map;
+		bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
+		    BUS_DMASYNC_PREWRITE);
+
+		/* ctrl segment */
+		sqe->sqe_opcode_index = betoh32(MCX_SQE_WQE_OPCODE_SEND |
+		    ((sc->sc_tx_prod & 0xffff) << MCX_SQE_WQE_INDEX_SHIFT));
+		sqe->sqe_ds_sq_num = betoh32((sc->sc_sqn << MCX_SQE_SQ_NUM_SHIFT) |
+		    4 /* always use 64 byte sqes */);
+		/* always generate a completion event */
+		sqe->sqe_signature = betoh32(MCX_SQE_CE_CQE_ALWAYS);
+
+		/* eth segment */
+		sqe->sqe_inline_header_size = htobe16(16);
+		memcpy(sqe->sqe_inline_headers, m->m_data, 16);
+
+		/* data segment */
+		sqe->sqe_byte_count = betoh32(map->dm_mapsize - 16);
+		sqe->sqe_lkey = betoh32(sc->sc_lkey);
+		sqe->sqe_addr = betoh64(map->dm_segs[0].ds_addr + 16);
+
+		/* write the first 64 bits to the blue flame buffer */
+		bf = (uint64_t *)sqe;
+		bus_space_write_raw_8(sc->sc_memt, sc->sc_memh, bf_base + sc->sc_bf_offset, *bf);
+		/* next write goes to the other buffer */
+		sc->sc_bf_offset ^= sc->sc_bf_size;
+
+		idx++;
+		if (idx == (1 << MCX_LOG_SQ_SIZE))
+			idx = 0;
+		sc->sc_tx_prod++;
+		used++;
+	}
+
+	if (used)
+		*sc->sc_tx_doorbell = betoh32(sc->sc_tx_prod);
+}
+
+static void
+mcx_watchdog(struct ifnet *ifp)
+{
+}
+
+static void
+mcx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
+{
+	struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
+	struct mcx_reg_ptys ptys;
+	int i;
+	uint32_t proto_cap, proto_admin, proto_oper;
+	uint64_t media_oper;
+	uint64_t eth_cap_map[] = {
+		IFM_1000_SGMII,
+		IFM_1000_KX,
+		IFM_10G_CX4,
+		IFM_10G_KX4,
+		IFM_10G_KR,
+		0,
+		IFM_40G_CR4,
+		IFM_40G_KR4,
+		0,
+		0,
+		0,
+		0,
+		IFM_10G_SFP_CU,
+		IFM_10G_SR,
+		IFM_10G_LR,
+		IFM_40G_SR4,
+		IFM_40G_LR4,
+		0,
+		0, /* IFM_50G_SR2 */
+		0,
+		IFM_100G_CR4,
+		IFM_100G_SR4,
+		IFM_100G_KR4,
+		0,
+		0,
+		0,
+		0,
+		IFM_25G_CR,
+		IFM_25G_KR,
+		IFM_25G_SR,
+		IFM_50G_CR2,
+		IFM_50G_KR2
+	};
+
+	memset(&ptys, 0, sizeof(ptys));
+	ptys.rp_local_port = 1;
+	ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
+
+	if (mcx_read_hca_reg(sc, MCX_REG_PTYS, &ptys, sizeof(ptys)) != 0) {
+		printf("%s: unable to read port type/speed\n", DEVNAME(sc));
+		return;
+	}
+
+	proto_cap = betoh32(ptys.rp_eth_proto_cap);
+	proto_admin = betoh32(ptys.rp_eth_proto_admin);
+	proto_oper = betoh32(ptys.rp_eth_proto_oper);
+
+	ifmedia_delete_instance(&sc->sc_media, IFM_INST_ANY);
+	ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL);
+	ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO);
+	media_oper = 0;
+	for (i = 0; i < nitems(eth_cap_map); i++) {
+		if (proto_cap & (1 << i))
+			ifmedia_add(&sc->sc_media, IFM_ETHER | eth_cap_map[i], 0, NULL);
+
+		if (proto_oper & (1 << i)) {
+			media_oper = eth_cap_map[i];
+		}
+	}
+
+	if (ifmr != NULL) {
+		ifmr->ifm_status = IFM_AVALID;
+		/* not sure if this is the right thing to check, maybe paos? */
+		if (proto_oper != 0) {
+			ifmr->ifm_status |= IFM_ACTIVE;
+			ifmr->ifm_active = IFM_ETHER | IFM_AUTO |media_oper;
+			/* txpause, rxpause, duplex? */
+		}
+	}
+}
+
+static int
+mcx_media_change(struct ifnet *ifp)
+{
+	return (ENOTTY);
+}
+
+
+static inline uint32_t
+mcx_rd(struct mcx_softc *sc, bus_size_t r)
+{
+	uint32_t word;
+
+	word = bus_space_read_raw_4(sc->sc_memt, sc->sc_memh, r);
+
+	return (betoh32(word));
+}
+
+static inline void
+mcx_wr(struct mcx_softc *sc, bus_size_t r, uint32_t v)
+{
+	bus_space_write_raw_4(sc->sc_memt, sc->sc_memh, r, htobe32(v));
+}
+
+static inline void
+mcx_bar(struct mcx_softc *sc, bus_size_t r, bus_size_t l, int f)
+{
+	bus_space_barrier(sc->sc_memt, sc->sc_memh, r, l, f);
+}
+
+static int
+mcx_dmamem_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm,
+    bus_size_t size, u_int align)
+{
+	mxm->mxm_size = size;
+
+	if (bus_dmamap_create(sc->sc_dmat, mxm->mxm_size, 1,
+	    mxm->mxm_size, 0,
+	    BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT,
+	    &mxm->mxm_map) != 0)
+		return (1);
+	if (bus_dmamem_alloc(sc->sc_dmat, mxm->mxm_size,
+	    align, 0, &mxm->mxm_seg, 1, &mxm->mxm_nsegs,
+	    BUS_DMA_WAITOK | BUS_DMA_ZERO) != 0)
+		goto destroy;
+	if (bus_dmamem_map(sc->sc_dmat, &mxm->mxm_seg, mxm->mxm_nsegs,
+	    mxm->mxm_size, &mxm->mxm_kva, BUS_DMA_WAITOK) != 0)
+		goto free;
+	if (bus_dmamap_load(sc->sc_dmat, mxm->mxm_map, mxm->mxm_kva,
+	    mxm->mxm_size, NULL, BUS_DMA_WAITOK) != 0)
+		goto unmap;
+
+	return (0);
+unmap:
+	bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size);
+free:
+	bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1);
+destroy:
+	bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map);
+	return (1);
+}
+
+static void
+mcx_dmamem_zero(struct mcx_dmamem *mxm)
+{
+	memset(MCX_DMA_KVA(mxm), 0, MCX_DMA_LEN(mxm));
+}
+
+static void
+mcx_dmamem_free(struct mcx_softc *sc, struct mcx_dmamem *mxm)
+{
+	bus_dmamap_unload(sc->sc_dmat, mxm->mxm_map);
+	bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size);
+	bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1);
+	bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map);
+}
+
+static int
+mcx_hwmem_alloc(struct mcx_softc *sc, struct mcx_hwmem *mhm, unsigned int pages)
+{
+	bus_dma_segment_t *segs;
+	bus_size_t len = pages * MCX_PAGE_SIZE;
+	size_t seglen;
+
+	segs = mallocarray(sizeof(*segs), pages, M_DEVBUF, M_WAITOK|M_CANFAIL);
+	if (segs == NULL)
+		return (-1);
+
+	seglen = sizeof(*segs) * pages;
+
+	if (bus_dmamem_alloc(sc->sc_dmat, len, MCX_PAGE_SIZE, 0,
+	    segs, pages, &mhm->mhm_seg_count, BUS_DMA_NOWAIT) != 0)
+		goto free_segs;
+
+	if (mhm->mhm_seg_count < pages) {
+		size_t nseglen;
+
+		mhm->mhm_segs = mallocarray(sizeof(*mhm->mhm_segs),
+		    mhm->mhm_seg_count, M_DEVBUF, M_WAITOK|M_CANFAIL);
+		if (mhm->mhm_segs == NULL)
+			goto free_dmamem;
+
+		nseglen = sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count;
+
+		memcpy(mhm->mhm_segs, segs, nseglen);
+
+		free(segs, M_DEVBUF, seglen);
+
+		segs = mhm->mhm_segs;
+		seglen = nseglen;
+	} else
+		mhm->mhm_segs = segs;
+
+	if (bus_dmamap_create(sc->sc_dmat, len, pages, MCX_PAGE_SIZE,
+	    MCX_PAGE_SIZE, BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW /*|BUS_DMA_64BIT*/,
+	    &mhm->mhm_map) != 0)
+		goto free_dmamem;
+
+	if (bus_dmamap_load_raw(sc->sc_dmat, mhm->mhm_map,
+	    mhm->mhm_segs, mhm->mhm_seg_count, len, BUS_DMA_NOWAIT) != 0)
+		goto destroy;
+
+	bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map,
+	    0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_PRERW);
+
+	mhm->mhm_npages = pages;
+
+	return (0);
+
+destroy:
+	bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map);
+free_dmamem:
+	bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count);
+free_segs:
+	free(segs, M_DEVBUF, seglen);
+	mhm->mhm_segs = NULL;
+
+	return (-1);
+}
+
+static void
+mcx_hwmem_free(struct mcx_softc *sc, struct mcx_hwmem *mhm)
+{
+	if (mhm->mhm_npages == 0)
+		return;
+
+	bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map,
+	    0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_POSTRW);
+
+	bus_dmamap_unload(sc->sc_dmat, mhm->mhm_map);
+	bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map);
+	bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count);
+	free(mhm->mhm_segs, M_DEVBUF,
+	    sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count);
+
+	mhm->mhm_npages = 0;
+}