path: root/sys/dev/pci/if_mcx.c

/*	$OpenBSD: if_mcx.c,v 1.115 2024/05/24 06:02:53 jsg Exp $ */

/*
 * Copyright (c) 2017 David Gwynne <dlg@openbsd.org>
 * Copyright (c) 2019 Jonathan Matthew <jmatthew@openbsd.org>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include "bpfilter.h"
#include "vlan.h"
#include "kstat.h"

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/sockio.h>
#include <sys/mbuf.h>
#include <sys/socket.h>
#include <sys/device.h>
#include <sys/timeout.h>
#include <sys/task.h>
#include <sys/atomic.h>
#include <sys/timetc.h>
#include <sys/intrmap.h>

#include <machine/bus.h>
#include <machine/intr.h>

#include <net/if.h>
#include <net/if_media.h>
#include <net/toeplitz.h>

#if NBPFILTER > 0
#include <net/bpf.h>
#endif

#if NKSTAT > 0
#include <sys/kstat.h>
#endif

#include <netinet/in.h>
#include <netinet/if_ether.h>

#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <dev/pci/pcidevs.h>

#define BUS_DMASYNC_PRERW	(BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)
#define BUS_DMASYNC_POSTRW	(BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE)

#define MCX_HCA_BAR	PCI_MAPREG_START /* BAR 0 */

#define MCX_FW_VER			0x0000
#define  MCX_FW_VER_MAJOR(_v)			((_v) & 0xffff)
#define  MCX_FW_VER_MINOR(_v)			((_v) >> 16)
#define MCX_CMDIF_FW_SUBVER		0x0004
#define  MCX_FW_VER_SUBMINOR(_v)		((_v) & 0xffff)
#define  MCX_CMDIF(_v)				((_v) >> 16)

#define MCX_ISSI			1 /* as per the PRM */
#define MCX_CMD_IF_SUPPORTED		5

#define MCX_HARDMTU			9500

enum mcx_cmdq_slot {
	MCX_CMDQ_SLOT_POLL = 0,
	MCX_CMDQ_SLOT_IOCTL,
	MCX_CMDQ_SLOT_KSTAT,
	MCX_CMDQ_SLOT_LINK,

	MCX_CMDQ_NUM_SLOTS
};

#define MCX_PAGE_SHIFT			12
#define MCX_PAGE_SIZE			(1 << MCX_PAGE_SHIFT)

/* queue sizes */
#define MCX_LOG_EQ_SIZE			7
#define MCX_LOG_CQ_SIZE			12
#define MCX_LOG_RQ_SIZE			10
#define MCX_LOG_SQ_SIZE			11

#define MCX_MAX_QUEUES			16

/* completion event moderation - about 10khz, or 90% of the cq */
#define MCX_CQ_MOD_PERIOD		50
#define MCX_CQ_MOD_COUNTER		\
	(((1 << (MCX_LOG_CQ_SIZE - 1)) * 9) / 10)

#define MCX_LOG_SQ_ENTRY_SIZE		6
#define MCX_SQ_ENTRY_MAX_SLOTS		4
#define MCX_SQ_SEGS_PER_SLOT		\
	(sizeof(struct mcx_sq_entry) / sizeof(struct mcx_sq_entry_seg))
#define MCX_SQ_MAX_SEGMENTS		\
	1 + ((MCX_SQ_ENTRY_MAX_SLOTS-1) * MCX_SQ_SEGS_PER_SLOT)

#define MCX_LOG_FLOW_TABLE_SIZE		5
#define MCX_NUM_STATIC_FLOWS		4 /* promisc, allmulti, ucast, bcast */
#define MCX_NUM_MCAST_FLOWS 		\
	((1 << MCX_LOG_FLOW_TABLE_SIZE) - MCX_NUM_STATIC_FLOWS)

#define MCX_SQ_INLINE_SIZE		18
CTASSERT(ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN == MCX_SQ_INLINE_SIZE);

/* doorbell offsets */
#define MCX_DOORBELL_AREA_SIZE		MCX_PAGE_SIZE

#define MCX_CQ_DOORBELL_BASE		0
#define MCX_CQ_DOORBELL_STRIDE		64

#define MCX_WQ_DOORBELL_BASE		MCX_PAGE_SIZE/2
#define MCX_WQ_DOORBELL_STRIDE		64
/* make sure the doorbells fit */
CTASSERT(MCX_MAX_QUEUES * MCX_CQ_DOORBELL_STRIDE < MCX_WQ_DOORBELL_BASE);
CTASSERT(MCX_MAX_QUEUES * MCX_WQ_DOORBELL_STRIDE <
    MCX_DOORBELL_AREA_SIZE - MCX_WQ_DOORBELL_BASE);

#define MCX_WQ_DOORBELL_MASK		0xffff

/* uar registers */
#define MCX_UAR_CQ_DOORBELL		0x20
#define MCX_UAR_EQ_DOORBELL_ARM		0x40
#define MCX_UAR_EQ_DOORBELL		0x48
#define MCX_UAR_BF			0x800

#define MCX_CMDQ_ADDR_HI		0x0010
#define MCX_CMDQ_ADDR_LO		0x0014
#define MCX_CMDQ_ADDR_NMASK		0xfff
#define MCX_CMDQ_LOG_SIZE(_v)		((_v) >> 4 & 0xf)
#define MCX_CMDQ_LOG_STRIDE(_v)		((_v) >> 0 & 0xf)
#define MCX_CMDQ_INTERFACE_MASK		(0x3 << 8)
#define MCX_CMDQ_INTERFACE_FULL_DRIVER	(0x0 << 8)
#define MCX_CMDQ_INTERFACE_DISABLED	(0x1 << 8)

#define MCX_CMDQ_DOORBELL		0x0018

#define MCX_STATE			0x01fc
#define MCX_STATE_MASK				(1U << 31)
#define MCX_STATE_INITIALIZING			(1U << 31)
#define MCX_STATE_READY				(0 << 31)
#define MCX_STATE_INTERFACE_MASK		(0x3 << 24)
#define MCX_STATE_INTERFACE_FULL_DRIVER		(0x0 << 24)
#define MCX_STATE_INTERFACE_DISABLED		(0x1 << 24)

#define MCX_INTERNAL_TIMER		0x1000
#define MCX_INTERNAL_TIMER_H		0x1000
#define MCX_INTERNAL_TIMER_L		0x1004

#define MCX_CLEAR_INT			0x100c

#define MCX_REG_OP_WRITE		0
#define MCX_REG_OP_READ			1

#define MCX_REG_PMLP			0x5002
#define MCX_REG_PMTU			0x5003
#define MCX_REG_PTYS			0x5004
#define MCX_REG_PAOS			0x5006
#define MCX_REG_PFCC			0x5007
#define MCX_REG_PPCNT			0x5008
#define MCX_REG_MTCAP			0x9009 /* mgmt temp capabilities */
#define MCX_REG_MTMP			0x900a /* mgmt temp */
#define MCX_REG_MCIA			0x9014
#define MCX_REG_MCAM			0x907f

#define MCX_ETHER_CAP_SGMII		0
#define MCX_ETHER_CAP_1000_KX		1
#define MCX_ETHER_CAP_10G_CX4		2
#define MCX_ETHER_CAP_10G_KX4		3
#define MCX_ETHER_CAP_10G_KR		4
#define MCX_ETHER_CAP_40G_CR4		6
#define MCX_ETHER_CAP_40G_KR4		7
#define MCX_ETHER_CAP_10G_CR		12
#define MCX_ETHER_CAP_10G_SR		13
#define MCX_ETHER_CAP_10G_LR		14
#define MCX_ETHER_CAP_40G_SR4		15
#define MCX_ETHER_CAP_40G_LR4		16
#define MCX_ETHER_CAP_50G_SR2		18
#define MCX_ETHER_CAP_100G_CR4		20
#define MCX_ETHER_CAP_100G_SR4		21
#define MCX_ETHER_CAP_100G_KR4		22
#define MCX_ETHER_CAP_100G_LR4		23
#define MCX_ETHER_CAP_25G_CR		27
#define MCX_ETHER_CAP_25G_KR		28
#define MCX_ETHER_CAP_25G_SR		29
#define MCX_ETHER_CAP_50G_CR2		30
#define MCX_ETHER_CAP_50G_KR2		31

#define MCX_ETHER_EXT_CAP_SGMII_100	0
#define MCX_ETHER_EXT_CAP_1000_X	1
#define MCX_ETHER_EXT_CAP_5G_R		3
#define MCX_ETHER_EXT_CAP_XAUI		4
#define MCX_ETHER_EXT_CAP_XLAUI		5
#define MCX_ETHER_EXT_CAP_25G_AUI1	6
#define MCX_ETHER_EXT_CAP_50G_AUI2	7
#define MCX_ETHER_EXT_CAP_50G_AUI1	8
#define MCX_ETHER_EXT_CAP_CAUI4		9
#define MCX_ETHER_EXT_CAP_100G_AUI2	10
#define MCX_ETHER_EXT_CAP_200G_AUI4	12
#define MCX_ETHER_EXT_CAP_400G_AUI8	15

#define MCX_MAX_CQE			32

#define MCX_CMD_QUERY_HCA_CAP		0x100
#define MCX_CMD_QUERY_ADAPTER		0x101
#define MCX_CMD_INIT_HCA		0x102
#define MCX_CMD_TEARDOWN_HCA		0x103
#define MCX_CMD_ENABLE_HCA		0x104
#define MCX_CMD_DISABLE_HCA		0x105
#define MCX_CMD_QUERY_PAGES		0x107
#define MCX_CMD_MANAGE_PAGES		0x108
#define MCX_CMD_SET_HCA_CAP		0x109
#define MCX_CMD_QUERY_ISSI		0x10a
#define MCX_CMD_SET_ISSI		0x10b
#define MCX_CMD_SET_DRIVER_VERSION	0x10d
#define MCX_CMD_QUERY_SPECIAL_CONTEXTS	0x203
#define MCX_CMD_CREATE_EQ		0x301
#define MCX_CMD_DESTROY_EQ		0x302
#define MCX_CMD_QUERY_EQ		0x303
#define MCX_CMD_CREATE_CQ		0x400
#define MCX_CMD_DESTROY_CQ		0x401
#define MCX_CMD_QUERY_CQ		0x402
#define MCX_CMD_QUERY_NIC_VPORT_CONTEXT	0x754
#define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT \
					0x755
#define MCX_CMD_QUERY_VPORT_COUNTERS	0x770
#define MCX_CMD_ALLOC_PD		0x800
#define MCX_CMD_ALLOC_UAR		0x802
#define MCX_CMD_ACCESS_REG		0x805
#define MCX_CMD_ALLOC_TRANSPORT_DOMAIN	0x816
#define MCX_CMD_CREATE_TIR		0x900
#define MCX_CMD_DESTROY_TIR		0x902
#define MCX_CMD_CREATE_SQ		0x904
#define MCX_CMD_MODIFY_SQ		0x905
#define MCX_CMD_DESTROY_SQ		0x906
#define MCX_CMD_QUERY_SQ		0x907
#define MCX_CMD_CREATE_RQ		0x908
#define MCX_CMD_MODIFY_RQ		0x909
#define MCX_CMD_DESTROY_RQ		0x90a
#define MCX_CMD_QUERY_RQ		0x90b
#define MCX_CMD_CREATE_TIS		0x912
#define MCX_CMD_DESTROY_TIS		0x914
#define MCX_CMD_CREATE_RQT		0x916
#define MCX_CMD_DESTROY_RQT		0x918
#define MCX_CMD_SET_FLOW_TABLE_ROOT	0x92f
#define MCX_CMD_CREATE_FLOW_TABLE	0x930
#define MCX_CMD_DESTROY_FLOW_TABLE	0x931
#define MCX_CMD_QUERY_FLOW_TABLE	0x932
#define MCX_CMD_CREATE_FLOW_GROUP	0x933
#define MCX_CMD_DESTROY_FLOW_GROUP	0x934
#define MCX_CMD_QUERY_FLOW_GROUP	0x935
#define MCX_CMD_SET_FLOW_TABLE_ENTRY	0x936
#define MCX_CMD_QUERY_FLOW_TABLE_ENTRY	0x937
#define MCX_CMD_DELETE_FLOW_TABLE_ENTRY	0x938
#define MCX_CMD_ALLOC_FLOW_COUNTER	0x939
#define MCX_CMD_QUERY_FLOW_COUNTER	0x93b

#define MCX_QUEUE_STATE_RST		0
#define MCX_QUEUE_STATE_RDY		1
#define MCX_QUEUE_STATE_ERR		3

#define MCX_FLOW_TABLE_TYPE_RX		0
#define MCX_FLOW_TABLE_TYPE_TX		1

#define MCX_CMDQ_INLINE_DATASIZE 16

struct mcx_cmdq_entry {
	uint8_t			cq_type;
#define MCX_CMDQ_TYPE_PCIE		0x7
	uint8_t			cq_reserved0[3];

	uint32_t		cq_input_length;
	uint64_t		cq_input_ptr;
	uint8_t			cq_input_data[MCX_CMDQ_INLINE_DATASIZE];

	uint8_t			cq_output_data[MCX_CMDQ_INLINE_DATASIZE];
	uint64_t		cq_output_ptr;
	uint32_t		cq_output_length;

	uint8_t			cq_token;
	uint8_t			cq_signature;
	uint8_t			cq_reserved1[1];
	uint8_t			cq_status;
#define MCX_CQ_STATUS_SHIFT		1
#define MCX_CQ_STATUS_MASK		(0x7f << MCX_CQ_STATUS_SHIFT)
#define MCX_CQ_STATUS_OK		(0x00 << MCX_CQ_STATUS_SHIFT)
#define MCX_CQ_STATUS_INT_ERR		(0x01 << MCX_CQ_STATUS_SHIFT)
#define MCX_CQ_STATUS_BAD_OPCODE	(0x02 << MCX_CQ_STATUS_SHIFT)
#define MCX_CQ_STATUS_BAD_PARAM		(0x03 << MCX_CQ_STATUS_SHIFT)
#define MCX_CQ_STATUS_BAD_SYS_STATE	(0x04 << MCX_CQ_STATUS_SHIFT)
#define MCX_CQ_STATUS_BAD_RESOURCE	(0x05 << MCX_CQ_STATUS_SHIFT)
#define MCX_CQ_STATUS_RESOURCE_BUSY	(0x06 << MCX_CQ_STATUS_SHIFT)
#define MCX_CQ_STATUS_EXCEED_LIM	(0x08 << MCX_CQ_STATUS_SHIFT)
#define MCX_CQ_STATUS_BAD_RES_STATE	(0x09 << MCX_CQ_STATUS_SHIFT)
#define MCX_CQ_STATUS_BAD_INDEX		(0x0a << MCX_CQ_STATUS_SHIFT)
#define MCX_CQ_STATUS_NO_RESOURCES	(0x0f << MCX_CQ_STATUS_SHIFT)
#define MCX_CQ_STATUS_BAD_INPUT_LEN	(0x50 << MCX_CQ_STATUS_SHIFT)
#define MCX_CQ_STATUS_BAD_OUTPUT_LEN	(0x51 << MCX_CQ_STATUS_SHIFT)
#define MCX_CQ_STATUS_BAD_RESOURCE_STATE \
					(0x10 << MCX_CQ_STATUS_SHIFT)
#define MCX_CQ_STATUS_BAD_SIZE		(0x40 << MCX_CQ_STATUS_SHIFT)
#define MCX_CQ_STATUS_OWN_MASK		0x1
#define MCX_CQ_STATUS_OWN_SW		0x0
#define MCX_CQ_STATUS_OWN_HW		0x1
} __packed __aligned(8);

#define MCX_CMDQ_MAILBOX_DATASIZE	512

struct mcx_cmdq_mailbox {
	uint8_t			mb_data[MCX_CMDQ_MAILBOX_DATASIZE];
	uint8_t			mb_reserved0[48];
	uint64_t		mb_next_ptr;
	uint32_t		mb_block_number;
	uint8_t			mb_reserved1[1];
	uint8_t			mb_token;
	uint8_t			mb_ctrl_signature;
	uint8_t			mb_signature;
} __packed __aligned(8);

#define MCX_CMDQ_MAILBOX_ALIGN	(1 << 10)
#define MCX_CMDQ_MAILBOX_SIZE	roundup(sizeof(struct mcx_cmdq_mailbox), \
				    MCX_CMDQ_MAILBOX_ALIGN)
/*
 * command mailbox structures
 */

struct mcx_cmd_enable_hca_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[2];
	uint16_t		cmd_function_id;
	uint8_t			cmd_reserved2[4];
} __packed __aligned(4);

struct mcx_cmd_enable_hca_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_cmd_init_hca_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_init_hca_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_teardown_hca_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[2];
#define MCX_CMD_TEARDOWN_HCA_GRACEFUL	0x0
#define MCX_CMD_TEARDOWN_HCA_PANIC	0x1
	uint16_t		cmd_profile;
	uint8_t			cmd_reserved2[4];
} __packed __aligned(4);

struct mcx_cmd_teardown_hca_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_access_reg_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[2];
	uint16_t		cmd_register_id;
	uint32_t		cmd_argument;
} __packed __aligned(4);

struct mcx_cmd_access_reg_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_reg_pmtu {
	uint8_t			rp_reserved1;
	uint8_t			rp_local_port;
	uint8_t			rp_reserved2[2];
	uint16_t		rp_max_mtu;
	uint8_t			rp_reserved3[2];
	uint16_t		rp_admin_mtu;
	uint8_t			rp_reserved4[2];
	uint16_t		rp_oper_mtu;
	uint8_t			rp_reserved5[2];
} __packed __aligned(4);

struct mcx_reg_ptys {
	uint8_t			rp_reserved1;
	uint8_t			rp_local_port;
	uint8_t			rp_reserved2;
	uint8_t			rp_proto_mask;
#define MCX_REG_PTYS_PROTO_MASK_ETH		(1 << 2)
	uint8_t			rp_reserved3[4];
	uint32_t		rp_ext_eth_proto_cap;
	uint32_t		rp_eth_proto_cap;
	uint8_t			rp_reserved4[4];
	uint32_t		rp_ext_eth_proto_admin;
	uint32_t		rp_eth_proto_admin;
	uint8_t			rp_reserved5[4];
	uint32_t		rp_ext_eth_proto_oper;
	uint32_t		rp_eth_proto_oper;
	uint8_t			rp_reserved6[24];
} __packed __aligned(4);

struct mcx_reg_paos {
	uint8_t			rp_reserved1;
	uint8_t			rp_local_port;
	uint8_t			rp_admin_status;
#define MCX_REG_PAOS_ADMIN_STATUS_UP		1
#define MCX_REG_PAOS_ADMIN_STATUS_DOWN		2
#define MCX_REG_PAOS_ADMIN_STATUS_UP_ONCE	3
#define MCX_REG_PAOS_ADMIN_STATUS_DISABLED	4
	uint8_t			rp_oper_status;
#define MCX_REG_PAOS_OPER_STATUS_UP		1
#define MCX_REG_PAOS_OPER_STATUS_DOWN		2
#define MCX_REG_PAOS_OPER_STATUS_FAILED		4
	uint8_t			rp_admin_state_update;
#define MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN	(1 << 7)
	uint8_t			rp_reserved2[11];
} __packed __aligned(4);

struct mcx_reg_pfcc {
	uint8_t			rp_reserved1;
	uint8_t			rp_local_port;
	uint8_t			rp_reserved2[3];
	uint8_t			rp_prio_mask_tx;
	uint8_t			rp_reserved3;
	uint8_t			rp_prio_mask_rx;
	uint8_t			rp_pptx_aptx;
	uint8_t			rp_pfctx;
	uint8_t			rp_fctx_dis;
	uint8_t			rp_reserved4;
	uint8_t			rp_pprx_aprx;
	uint8_t			rp_pfcrx;
	uint8_t			rp_reserved5[2];
	uint16_t		rp_dev_stall_min;
	uint16_t		rp_dev_stall_crit;
	uint8_t			rp_reserved6[12];
} __packed __aligned(4);

#define MCX_PMLP_MODULE_NUM_MASK	0xff
struct mcx_reg_pmlp {
	uint8_t			rp_rxtx;
	uint8_t			rp_local_port;
	uint8_t			rp_reserved0;
	uint8_t			rp_width;
	uint32_t		rp_lane0_mapping;
	uint32_t		rp_lane1_mapping;
	uint32_t		rp_lane2_mapping;
	uint32_t		rp_lane3_mapping;
	uint8_t			rp_reserved1[44];
} __packed __aligned(4);

struct mcx_reg_ppcnt {
	uint8_t			ppcnt_swid;
	uint8_t			ppcnt_local_port;
	uint8_t			ppcnt_pnat;
	uint8_t			ppcnt_grp;
#define MCX_REG_PPCNT_GRP_IEEE8023		0x00
#define MCX_REG_PPCNT_GRP_RFC2863		0x01
#define MCX_REG_PPCNT_GRP_RFC2819		0x02
#define MCX_REG_PPCNT_GRP_RFC3635		0x03
#define MCX_REG_PPCNT_GRP_PER_PRIO		0x10
#define MCX_REG_PPCNT_GRP_PER_TC		0x11
#define MCX_REG_PPCNT_GRP_PER_RX_BUFFER		0x11

	uint8_t			ppcnt_clr;
	uint8_t			ppcnt_reserved1[2];
	uint8_t			ppcnt_prio_tc;
#define MCX_REG_PPCNT_CLR			(1 << 7)

	uint8_t			ppcnt_counter_set[248];
} __packed __aligned(8);
CTASSERT(sizeof(struct mcx_reg_ppcnt) == 256);
CTASSERT((offsetof(struct mcx_reg_ppcnt, ppcnt_counter_set) %
    sizeof(uint64_t)) == 0);

enum mcx_ppcnt_ieee8023 {
	frames_transmitted_ok,
	frames_received_ok,
	frame_check_sequence_errors,
	alignment_errors,
	octets_transmitted_ok,
	octets_received_ok,
	multicast_frames_xmitted_ok,
	broadcast_frames_xmitted_ok,
	multicast_frames_received_ok,
	broadcast_frames_received_ok,
	in_range_length_errors,
	out_of_range_length_field,
	frame_too_long_errors,
	symbol_error_during_carrier,
	mac_control_frames_transmitted,
	mac_control_frames_received,
	unsupported_opcodes_received,
	pause_mac_ctrl_frames_received,
	pause_mac_ctrl_frames_transmitted,

	mcx_ppcnt_ieee8023_count
};
CTASSERT(mcx_ppcnt_ieee8023_count * sizeof(uint64_t) == 0x98);

enum mcx_ppcnt_rfc2863 {
	in_octets,
	in_ucast_pkts,
	in_discards,
	in_errors,
	in_unknown_protos,
	out_octets,
	out_ucast_pkts,
	out_discards,
	out_errors,
	in_multicast_pkts,
	in_broadcast_pkts,
	out_multicast_pkts,
	out_broadcast_pkts,

	mcx_ppcnt_rfc2863_count
};
CTASSERT(mcx_ppcnt_rfc2863_count * sizeof(uint64_t) == 0x68);

enum mcx_ppcnt_rfc2819 {
	drop_events,
	octets,
	pkts,
	broadcast_pkts,
	multicast_pkts,
	crc_align_errors,
	undersize_pkts,
	oversize_pkts,
	fragments,
	jabbers,
	collisions,
	pkts64octets,
	pkts65to127octets,
	pkts128to255octets,
	pkts256to511octets,
	pkts512to1023octets,
	pkts1024to1518octets,
	pkts1519to2047octets,
	pkts2048to4095octets,
	pkts4096to8191octets,
	pkts8192to10239octets,

	mcx_ppcnt_rfc2819_count
};
CTASSERT((mcx_ppcnt_rfc2819_count * sizeof(uint64_t)) == 0xa8);

enum mcx_ppcnt_rfc3635 {
	dot3stats_alignment_errors,
	dot3stats_fcs_errors,
	dot3stats_single_collision_frames,
	dot3stats_multiple_collision_frames,
	dot3stats_sqe_test_errors,
	dot3stats_deferred_transmissions,
	dot3stats_late_collisions,
	dot3stats_excessive_collisions,
	dot3stats_internal_mac_transmit_errors,
	dot3stats_carrier_sense_errors,
	dot3stats_frame_too_longs,
	dot3stats_internal_mac_receive_errors,
	dot3stats_symbol_errors,
	dot3control_in_unknown_opcodes,
	dot3in_pause_frames,
	dot3out_pause_frames,

	mcx_ppcnt_rfc3635_count
};
CTASSERT((mcx_ppcnt_rfc3635_count * sizeof(uint64_t)) == 0x80);

struct mcx_reg_mcam {
	uint8_t			_reserved1[1];
	uint8_t			mcam_feature_group;
	uint8_t			_reserved2[1];
	uint8_t			mcam_access_reg_group;
	uint8_t			_reserved3[4];
	uint8_t			mcam_access_reg_cap_mask[16];
	uint8_t			_reserved4[16];
	uint8_t			mcam_feature_cap_mask[16];
	uint8_t			_reserved5[16];
} __packed __aligned(4);

#define MCX_BITFIELD_BIT(bf, b)	(bf[(sizeof bf - 1) - (b / 8)] & (b % 8))

#define MCX_MCAM_FEATURE_CAP_SENSOR_MAP	6

struct mcx_reg_mtcap {
	uint8_t			_reserved1[3];
	uint8_t			mtcap_sensor_count;
	uint8_t			_reserved2[4];

	uint64_t		mtcap_sensor_map;
};

struct mcx_reg_mtmp {
	uint8_t			_reserved1[2];
	uint16_t		mtmp_sensor_index;

	uint8_t			_reserved2[2];
	uint16_t		mtmp_temperature;

	uint16_t		mtmp_mte_mtr;
#define MCX_REG_MTMP_MTE		(1 << 15)
#define MCX_REG_MTMP_MTR		(1 << 14)
	uint16_t		mtmp_max_temperature;

	uint16_t		mtmp_tee;
#define MCX_REG_MTMP_TEE_NOPE		(0 << 14)
#define MCX_REG_MTMP_TEE_GENERATE	(1 << 14)
#define MCX_REG_MTMP_TEE_GENERATE_ONE	(2 << 14)
	uint16_t		mtmp_temperature_threshold_hi;

	uint8_t			_reserved3[2];
	uint16_t		mtmp_temperature_threshold_lo;

	uint8_t			_reserved4[4];

	uint8_t			mtmp_sensor_name[8];
};
CTASSERT(sizeof(struct mcx_reg_mtmp) == 0x20);
CTASSERT(offsetof(struct mcx_reg_mtmp, mtmp_sensor_name) == 0x18);

#define MCX_MCIA_EEPROM_BYTES	32
struct mcx_reg_mcia {
	uint8_t			rm_l;
	uint8_t			rm_module;
	uint8_t			rm_reserved0;
	uint8_t			rm_status;
	uint8_t			rm_i2c_addr;
	uint8_t			rm_page_num;
	uint16_t		rm_dev_addr;
	uint16_t		rm_reserved1;
	uint16_t		rm_size;
	uint32_t		rm_reserved2;
	uint8_t			rm_data[48];
} __packed __aligned(4);

struct mcx_cmd_query_issi_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_query_issi_il_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[2];
	uint16_t		cmd_current_issi;
	uint8_t			cmd_reserved2[4];
} __packed __aligned(4);

CTASSERT(sizeof(struct mcx_cmd_query_issi_il_out) == MCX_CMDQ_INLINE_DATASIZE);

struct mcx_cmd_query_issi_mb_out {
	uint8_t			cmd_reserved2[16];
	uint8_t			cmd_supported_issi[80]; /* very big endian */
} __packed __aligned(4);

CTASSERT(sizeof(struct mcx_cmd_query_issi_mb_out) <= MCX_CMDQ_MAILBOX_DATASIZE);

struct mcx_cmd_set_issi_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[2];
	uint16_t		cmd_current_issi;
	uint8_t			cmd_reserved2[4];
} __packed __aligned(4);

CTASSERT(sizeof(struct mcx_cmd_set_issi_in) <= MCX_CMDQ_INLINE_DATASIZE);

struct mcx_cmd_set_issi_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

CTASSERT(sizeof(struct mcx_cmd_set_issi_out) <= MCX_CMDQ_INLINE_DATASIZE);

struct mcx_cmd_query_pages_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
#define MCX_CMD_QUERY_PAGES_BOOT	0x01
#define MCX_CMD_QUERY_PAGES_INIT	0x02
#define MCX_CMD_QUERY_PAGES_REGULAR	0x03
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_query_pages_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[2];
	uint16_t		cmd_func_id;
	int32_t			cmd_num_pages;
} __packed __aligned(4);

struct mcx_cmd_manage_pages_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
#define MCX_CMD_MANAGE_PAGES_ALLOC_FAIL \
					0x00
#define MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS \
					0x01
#define MCX_CMD_MANAGE_PAGES_HCA_RETURN_PAGES \
					0x02
	uint8_t			cmd_reserved1[2];
	uint16_t		cmd_func_id;
	uint32_t		cmd_input_num_entries;
} __packed __aligned(4);

CTASSERT(sizeof(struct mcx_cmd_manage_pages_in) == MCX_CMDQ_INLINE_DATASIZE);

struct mcx_cmd_manage_pages_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint32_t		cmd_output_num_entries;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

CTASSERT(sizeof(struct mcx_cmd_manage_pages_out) == MCX_CMDQ_INLINE_DATASIZE);

struct mcx_cmd_query_hca_cap_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
#define MCX_CMD_QUERY_HCA_CAP_MAX	(0x0 << 0)
#define MCX_CMD_QUERY_HCA_CAP_CURRENT	(0x1 << 0)
#define MCX_CMD_QUERY_HCA_CAP_DEVICE	(0x0 << 1)
#define MCX_CMD_QUERY_HCA_CAP_OFFLOAD	(0x1 << 1)
#define MCX_CMD_QUERY_HCA_CAP_FLOW	(0x7 << 1)
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_query_hca_cap_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

#define MCX_HCA_CAP_LEN			0x1000
#define MCX_HCA_CAP_NMAILBOXES		\
	(MCX_HCA_CAP_LEN / MCX_CMDQ_MAILBOX_DATASIZE)

#if __GNUC_PREREQ__(4, 3)
#define __counter__		__COUNTER__
#else
#define __counter__		__LINE__
#endif

#define __token(_tok, _num)	_tok##_num
#define _token(_tok, _num)	__token(_tok, _num)
#define __reserved__		_token(__reserved, __counter__)

struct mcx_cap_device {
	uint8_t			reserved0[16];

	uint8_t			log_max_srq_sz;
	uint8_t			log_max_qp_sz;
	uint8_t			__reserved__[1];
	uint8_t			log_max_qp; /* 5 bits */
#define MCX_CAP_DEVICE_LOG_MAX_QP	0x1f

	uint8_t			__reserved__[1];
	uint8_t			log_max_srq; /* 5 bits */
#define MCX_CAP_DEVICE_LOG_MAX_SRQ	0x1f
	uint8_t			__reserved__[2];

	uint8_t			__reserved__[1];
	uint8_t			log_max_cq_sz;
	uint8_t			__reserved__[1];
	uint8_t			log_max_cq; /* 5 bits */
#define MCX_CAP_DEVICE_LOG_MAX_CQ	0x1f

	uint8_t			log_max_eq_sz;
	uint8_t			log_max_mkey; /* 6 bits */
#define MCX_CAP_DEVICE_LOG_MAX_MKEY	0x3f
	uint8_t			__reserved__[1];
	uint8_t			log_max_eq; /* 4 bits */
#define MCX_CAP_DEVICE_LOG_MAX_EQ	0x0f

	uint8_t			max_indirection;
	uint8_t			log_max_mrw_sz; /* 7 bits */
#define MCX_CAP_DEVICE_LOG_MAX_MRW_SZ	0x7f
	uint8_t			teardown_log_max_msf_list_size;
#define MCX_CAP_DEVICE_FORCE_TEARDOWN	0x80
#define MCX_CAP_DEVICE_LOG_MAX_MSF_LIST_SIZE \
					0x3f
	uint8_t			log_max_klm_list_size; /* 6 bits */
#define MCX_CAP_DEVICE_LOG_MAX_KLM_LIST_SIZE \
					0x3f

	uint8_t			__reserved__[1];
	uint8_t			log_max_ra_req_dc; /* 6 bits */
#define MCX_CAP_DEVICE_LOG_MAX_REQ_DC	0x3f
	uint8_t			__reserved__[1];
	uint8_t			log_max_ra_res_dc; /* 6 bits */
#define MCX_CAP_DEVICE_LOG_MAX_RA_RES_DC \
					0x3f

	uint8_t			__reserved__[1];
	uint8_t			log_max_ra_req_qp; /* 6 bits */
#define MCX_CAP_DEVICE_LOG_MAX_RA_REQ_QP \
					0x3f
	uint8_t			__reserved__[1];
	uint8_t			log_max_ra_res_qp; /* 6 bits */
#define MCX_CAP_DEVICE_LOG_MAX_RA_RES_QP \
					0x3f

	uint8_t			flags1;
#define MCX_CAP_DEVICE_END_PAD		0x80
#define MCX_CAP_DEVICE_CC_QUERY_ALLOWED	0x40
#define MCX_CAP_DEVICE_CC_MODIFY_ALLOWED \
					0x20
#define MCX_CAP_DEVICE_START_PAD	0x10
#define MCX_CAP_DEVICE_128BYTE_CACHELINE \
					0x08
	uint8_t			__reserved__[1];
	uint16_t		gid_table_size;

	uint16_t		flags2;
#define MCX_CAP_DEVICE_OUT_OF_SEQ_CNT	0x8000
#define MCX_CAP_DEVICE_VPORT_COUNTERS	0x4000
#define MCX_CAP_DEVICE_RETRANSMISSION_Q_COUNTERS \
					0x2000
#define MCX_CAP_DEVICE_DEBUG		0x1000
#define MCX_CAP_DEVICE_MODIFY_RQ_COUNTERS_SET_ID \
					0x8000
#define MCX_CAP_DEVICE_RQ_DELAY_DROP	0x4000
#define MCX_CAP_DEVICe_MAX_QP_CNT_MASK	0x03ff
	uint16_t		pkey_table_size;

	uint8_t			flags3;
#define MCX_CAP_DEVICE_VPORT_GROUP_MANAGER \
					0x80
#define MCX_CAP_DEVICE_VHCA_GROUP_MANAGER \
					0x40
#define MCX_CAP_DEVICE_IB_VIRTUAL	0x20
#define MCX_CAP_DEVICE_ETH_VIRTUAL	0x10
#define MCX_CAP_DEVICE_ETS		0x04
#define MCX_CAP_DEVICE_NIC_FLOW_TABLE	0x02
#define MCX_CAP_DEVICE_ESWITCH_FLOW_TABLE \
					0x01
	uint8_t			local_ca_ack_delay; /* 5 bits */
#define MCX_CAP_DEVICE_LOCAL_CA_ACK_DELAY \
					0x1f
#define MCX_CAP_DEVICE_MCAM_REG		0x40
	uint8_t			port_type;
#define MCX_CAP_DEVICE_PORT_MODULE_EVENT \
					0x80
#define MCX_CAP_DEVICE_PORT_TYPE	0x03
#define MCX_CAP_DEVICE_PORT_TYPE_ETH	0x01
	uint8_t			num_ports;

	uint8_t			snapshot_log_max_msg;
#define MCX_CAP_DEVICE_SNAPSHOT		0x80
#define MCX_CAP_DEVICE_LOG_MAX_MSG	0x1f
	uint8_t			max_tc; /* 4 bits */
#define MCX_CAP_DEVICE_MAX_TC		0x0f
	uint8_t			flags4;
#define MCX_CAP_DEVICE_TEMP_WARN_EVENT	0x80
#define MCX_CAP_DEVICE_DCBX		0x40
#define MCX_CAP_DEVICE_ROL_S		0x02
#define MCX_CAP_DEVICE_ROL_G		0x01
	uint8_t			wol;
#define MCX_CAP_DEVICE_WOL_S		0x40
#define MCX_CAP_DEVICE_WOL_G		0x20
#define MCX_CAP_DEVICE_WOL_A		0x10
#define MCX_CAP_DEVICE_WOL_B		0x08
#define MCX_CAP_DEVICE_WOL_M		0x04
#define MCX_CAP_DEVICE_WOL_U		0x02
#define MCX_CAP_DEVICE_WOL_P		0x01

	uint16_t		stat_rate_support;
	uint8_t			__reserved__[1];
	uint8_t			cqe_version; /* 4 bits */
#define MCX_CAP_DEVICE_CQE_VERSION	0x0f

	uint32_t		flags5;
#define MCX_CAP_DEVICE_COMPACT_ADDRESS_VECTOR \
					0x80000000
#define MCX_CAP_DEVICE_STRIDING_RQ	0x40000000
#define MCX_CAP_DEVICE_IPOIP_ENHANCED_OFFLOADS \
					0x10000000
#define MCX_CAP_DEVICE_IPOIP_IPOIP_OFFLOADS \
					0x08000000
#define MCX_CAP_DEVICE_DC_CONNECT_CP	0x00040000
#define MCX_CAP_DEVICE_DC_CNAK_DRACE	0x00020000
#define MCX_CAP_DEVICE_DRAIN_SIGERR	0x00010000
#define MCX_CAP_DEVICE_DRAIN_SIGERR	0x00010000
#define MCX_CAP_DEVICE_CMDIF_CHECKSUM	0x0000c000
#define MCX_CAP_DEVICE_SIGERR_QCE	0x00002000
#define MCX_CAP_DEVICE_WQ_SIGNATURE	0x00000800
#define MCX_CAP_DEVICE_SCTR_DATA_CQE	0x00000400
#define MCX_CAP_DEVICE_SHO		0x00000100
#define MCX_CAP_DEVICE_TPH		0x00000080
#define MCX_CAP_DEVICE_RF		0x00000040
#define MCX_CAP_DEVICE_DCT		0x00000020
#define MCX_CAP_DEVICE_QOS		0x00000010
#define MCX_CAP_DEVICe_ETH_NET_OFFLOADS	0x00000008
#define MCX_CAP_DEVICE_ROCE		0x00000004
#define MCX_CAP_DEVICE_ATOMIC		0x00000002

	uint32_t		flags6;
#define MCX_CAP_DEVICE_CQ_OI		0x80000000
#define MCX_CAP_DEVICE_CQ_RESIZE	0x40000000
#define MCX_CAP_DEVICE_CQ_MODERATION	0x20000000
#define MCX_CAP_DEVICE_CQ_PERIOD_MODE_MODIFY \
					0x10000000
#define MCX_CAP_DEVICE_CQ_INVALIDATE	0x08000000
#define MCX_CAP_DEVICE_RESERVED_AT_255	0x04000000
#define MCX_CAP_DEVICE_CQ_EQ_REMAP	0x02000000
#define MCX_CAP_DEVICE_PG		0x01000000
#define MCX_CAP_DEVICE_BLOCK_LB_MC	0x00800000
#define MCX_CAP_DEVICE_EXPONENTIAL_BACKOFF \
					0x00400000
#define MCX_CAP_DEVICE_SCQE_BREAK_MODERATION \
					0x00200000
#define MCX_CAP_DEVICE_CQ_PERIOD_START_FROM_CQE \
					0x00100000
#define MCX_CAP_DEVICE_CD		0x00080000
#define MCX_CAP_DEVICE_ATM		0x00040000
#define MCX_CAP_DEVICE_APM		0x00020000
#define MCX_CAP_DEVICE_IMAICL		0x00010000
#define MCX_CAP_DEVICE_QKV		0x00000200
#define MCX_CAP_DEVICE_PKV		0x00000100
#define MCX_CAP_DEVICE_SET_DETH_SQPN	0x00000080
#define MCX_CAP_DEVICE_XRC		0x00000008
#define MCX_CAP_DEVICE_UD		0x00000004
#define MCX_CAP_DEVICE_UC		0x00000002
#define MCX_CAP_DEVICE_RC		0x00000001

	uint8_t			uar_flags;
#define MCX_CAP_DEVICE_UAR_4K		0x80
	uint8_t			uar_sz;	/* 6 bits */
#define MCX_CAP_DEVICE_UAR_SZ		0x3f
	uint8_t			__reserved__[1];
	uint8_t			log_pg_sz;

	uint8_t			flags7;
#define MCX_CAP_DEVICE_BF		0x80
#define MCX_CAP_DEVICE_DRIVER_VERSION	0x40
#define MCX_CAP_DEVICE_PAD_TX_ETH_PACKET \
					0x20
	uint8_t			log_bf_reg_size; /* 5 bits */
#define MCX_CAP_DEVICE_LOG_BF_REG_SIZE	0x1f
	uint8_t			__reserved__[2];

	uint16_t		num_of_diagnostic_counters;
	uint16_t		max_wqe_sz_sq;

	uint8_t			__reserved__[2];
	uint16_t		max_wqe_sz_rq;

	uint8_t			__reserved__[2];
	uint16_t		max_wqe_sz_sq_dc;

	uint32_t		max_qp_mcg; /* 25 bits */
#define MCX_CAP_DEVICE_MAX_QP_MCG	0x1ffffff

	uint8_t			__reserved__[3];
	uint8_t			log_max_mcq;

	uint8_t			log_max_transport_domain; /* 5 bits */
#define MCX_CAP_DEVICE_LOG_MAX_TRANSORT_DOMAIN \
					0x1f
	uint8_t			log_max_pd; /* 5 bits */
#define MCX_CAP_DEVICE_LOG_MAX_PD	0x1f
	uint8_t			__reserved__[1];
	uint8_t			log_max_xrcd; /* 5 bits */
#define MCX_CAP_DEVICE_LOG_MAX_XRCD	0x1f

	uint8_t			__reserved__[2];
	uint16_t		max_flow_counter;

	uint8_t			log_max_rq; /* 5 bits */
#define MCX_CAP_DEVICE_LOG_MAX_RQ	0x1f
	uint8_t			log_max_sq; /* 5 bits */
#define MCX_CAP_DEVICE_LOG_MAX_SQ	0x1f
	uint8_t			log_max_tir; /* 5 bits */
#define MCX_CAP_DEVICE_LOG_MAX_TIR	0x1f
	uint8_t			log_max_tis; /* 5 bits */
#define MCX_CAP_DEVICE_LOG_MAX_TIS	0x1f

	uint8_t 		flags8;
#define MCX_CAP_DEVICE_BASIC_CYCLIC_RCV_WQE \
					0x80
#define MCX_CAP_DEVICE_LOG_MAX_RMP	0x1f
	uint8_t			log_max_rqt; /* 5 bits */
#define MCX_CAP_DEVICE_LOG_MAX_RQT	0x1f
	uint8_t			log_max_rqt_size; /* 5 bits */
#define MCX_CAP_DEVICE_LOG_MAX_RQT_SIZE	0x1f
	uint8_t			log_max_tis_per_sq; /* 5 bits */
#define MCX_CAP_DEVICE_LOG_MAX_TIS_PER_SQ \
					0x1f

	uint8_t			flags9;
#define MXC_CAP_DEVICE_EXT_STRIDE_NUM_RANGES \
					0x80
#define MXC_CAP_DEVICE_LOG_MAX_STRIDE_SZ_RQ \
					0x1f
	uint8_t			log_min_stride_sz_rq; /* 5 bits */
#define MXC_CAP_DEVICE_LOG_MIN_STRIDE_SZ_RQ \
					0x1f
	uint8_t			log_max_stride_sz_sq; /* 5 bits */
#define MXC_CAP_DEVICE_LOG_MAX_STRIDE_SZ_SQ \
					0x1f
	uint8_t			log_min_stride_sz_sq; /* 5 bits */
#define MXC_CAP_DEVICE_LOG_MIN_STRIDE_SZ_SQ \
					0x1f

	uint8_t			log_max_hairpin_queues;
#define MXC_CAP_DEVICE_HAIRPIN		0x80
#define MXC_CAP_DEVICE_LOG_MAX_HAIRPIN_QUEUES \
					0x1f
	uint8_t			log_min_hairpin_queues;
#define MXC_CAP_DEVICE_LOG_MIN_HAIRPIN_QUEUES \
					0x1f
	uint8_t			log_max_hairpin_num_packets;
#define MXC_CAP_DEVICE_LOG_MAX_HAIRPIN_NUM_PACKETS \
					0x1f
	uint8_t			log_max_mq_sz;
#define MXC_CAP_DEVICE_LOG_MAX_WQ_SZ \
					0x1f

	uint8_t			log_min_hairpin_wq_data_sz;
#define MXC_CAP_DEVICE_NIC_VPORT_CHANGE_EVENT \
					0x80
#define MXC_CAP_DEVICE_DISABLE_LOCAL_LB_UC \
					0x40
#define MXC_CAP_DEVICE_DISABLE_LOCAL_LB_MC \
					0x20
#define MCX_CAP_DEVICE_LOG_MIN_HAIRPIN_WQ_DATA_SZ \
					0x1f
	uint8_t			log_max_vlan_list;
#define MXC_CAP_DEVICE_SYSTEM_IMAGE_GUID_MODIFIABLE \
					0x80
#define MXC_CAP_DEVICE_LOG_MAX_VLAN_LIST \
					0x1f
	uint8_t			log_max_current_mc_list;
#define MXC_CAP_DEVICE_LOG_MAX_CURRENT_MC_LIST \
					0x1f
	uint8_t			log_max_current_uc_list;
#define MXC_CAP_DEVICE_LOG_MAX_CURRENT_UC_LIST \
					0x1f

	uint8_t			__reserved__[4];

	uint32_t		create_qp_start_hint; /* 24 bits */

	uint8_t			log_max_uctx; /* 5 bits */
#define MXC_CAP_DEVICE_LOG_MAX_UCTX	0x1f
	uint8_t			log_max_umem; /* 5 bits */
#define MXC_CAP_DEVICE_LOG_MAX_UMEM	0x1f
	uint16_t		max_num_eqs;

	uint8_t			log_max_l2_table; /* 5 bits */
#define MXC_CAP_DEVICE_LOG_MAX_L2_TABLE	0x1f
	uint8_t			__reserved__[1];
	uint16_t		log_uar_page_sz;

	uint8_t			__reserved__[8];

	uint32_t		device_frequency_mhz;
	uint32_t		device_frequency_khz;
} __packed __aligned(8);

CTASSERT(offsetof(struct mcx_cap_device, max_indirection) == 0x20);
CTASSERT(offsetof(struct mcx_cap_device, flags1) == 0x2c);
CTASSERT(offsetof(struct mcx_cap_device, flags2) == 0x30);
CTASSERT(offsetof(struct mcx_cap_device, snapshot_log_max_msg) == 0x38);
CTASSERT(offsetof(struct mcx_cap_device, flags5) == 0x40);
CTASSERT(offsetof(struct mcx_cap_device, flags7) == 0x4c);
CTASSERT(offsetof(struct mcx_cap_device, device_frequency_mhz) == 0x98);
CTASSERT(offsetof(struct mcx_cap_device, device_frequency_khz) == 0x9c);
CTASSERT(sizeof(struct mcx_cap_device) <= MCX_CMDQ_MAILBOX_DATASIZE);

struct mcx_cmd_set_driver_version_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_set_driver_version_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_set_driver_version {
	uint8_t			cmd_driver_version[64];
} __packed __aligned(8);

struct mcx_cmd_modify_nic_vport_context_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[4];
	uint32_t		cmd_field_select;
#define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_ADDR	0x04
#define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC	0x10
#define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU	0x40
} __packed __aligned(4);

struct mcx_cmd_modify_nic_vport_context_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_query_nic_vport_context_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[4];
	uint8_t			cmd_allowed_list_type;
	uint8_t			cmd_reserved2[3];
} __packed __aligned(4);

struct mcx_cmd_query_nic_vport_context_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_nic_vport_ctx {
	uint32_t		vp_min_wqe_inline_mode;
	uint8_t			vp_reserved0[32];
	uint32_t		vp_mtu;
	uint8_t			vp_reserved1[200];
	uint16_t		vp_flags;
#define MCX_NIC_VPORT_CTX_LIST_UC_MAC			(0)
#define MCX_NIC_VPORT_CTX_LIST_MC_MAC			(1 << 24)
#define MCX_NIC_VPORT_CTX_LIST_VLAN			(2 << 24)
#define MCX_NIC_VPORT_CTX_PROMISC_ALL			(1 << 13)
#define MCX_NIC_VPORT_CTX_PROMISC_MCAST			(1 << 14)
#define MCX_NIC_VPORT_CTX_PROMISC_UCAST			(1 << 15)
	uint16_t		vp_allowed_list_size;
	uint64_t		vp_perm_addr;
	uint8_t			vp_reserved2[4];
	/* allowed list follows */
} __packed __aligned(4);

struct mcx_counter {
	uint64_t		packets;
	uint64_t		octets;
} __packed __aligned(4);

struct mcx_nic_vport_counters {
	struct mcx_counter	rx_err;
	struct mcx_counter	tx_err;
	uint8_t			reserved0[64]; /* 0x30 */
	struct mcx_counter	rx_bcast;
	struct mcx_counter	tx_bcast;
	struct mcx_counter	rx_ucast;
	struct mcx_counter	tx_ucast;
	struct mcx_counter	rx_mcast;
	struct mcx_counter	tx_mcast;
	uint8_t			reserved1[0x210 - 0xd0];
} __packed __aligned(4);

struct mcx_cmd_query_vport_counters_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_query_vport_counters_mb_in {
	uint8_t			cmd_reserved0[8];
	uint8_t			cmd_clear;
	uint8_t			cmd_reserved1[7];
} __packed __aligned(4);

struct mcx_cmd_query_vport_counters_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_query_flow_counter_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_query_flow_counter_mb_in {
	uint8_t			cmd_reserved0[8];
	uint8_t			cmd_clear;
	uint8_t			cmd_reserved1[5];
	uint16_t		cmd_flow_counter_id;
} __packed __aligned(4);

struct mcx_cmd_query_flow_counter_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_alloc_uar_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_alloc_uar_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint32_t		cmd_uar;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_cmd_query_special_ctx_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_query_special_ctx_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[4];
	uint32_t		cmd_resd_lkey;
} __packed __aligned(4);

struct mcx_eq_ctx {
	uint32_t		eq_status;
#define MCX_EQ_CTX_STATE_SHIFT		8
#define MCX_EQ_CTX_STATE_MASK		(0xf << MCX_EQ_CTX_STATE_SHIFT)
#define MCX_EQ_CTX_STATE_ARMED		0x9
#define MCX_EQ_CTX_STATE_FIRED		0xa
#define MCX_EQ_CTX_OI_SHIFT		17
#define MCX_EQ_CTX_OI			(1 << MCX_EQ_CTX_OI_SHIFT)
#define MCX_EQ_CTX_EC_SHIFT		18
#define MCX_EQ_CTX_EC			(1 << MCX_EQ_CTX_EC_SHIFT)
#define MCX_EQ_CTX_STATUS_SHIFT		28
#define MCX_EQ_CTX_STATUS_MASK		(0xf << MCX_EQ_CTX_STATUS_SHIFT)
#define MCX_EQ_CTX_STATUS_OK		0x0
#define MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE 0xa
	uint32_t		eq_reserved1;
	uint32_t		eq_page_offset;
#define MCX_EQ_CTX_PAGE_OFFSET_SHIFT	5
	uint32_t		eq_uar_size;
#define MCX_EQ_CTX_UAR_PAGE_MASK	0xffffff
#define MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT	24
	uint32_t		eq_reserved2;
	uint8_t			eq_reserved3[3];
	uint8_t			eq_intr;
	uint32_t		eq_log_page_size;
#define MCX_EQ_CTX_LOG_PAGE_SIZE_SHIFT	24
	uint32_t		eq_reserved4[3];
	uint32_t		eq_consumer_counter;
	uint32_t		eq_producer_counter;
#define MCX_EQ_CTX_COUNTER_MASK		0xffffff
	uint32_t		eq_reserved5[4];
} __packed __aligned(4);

CTASSERT(sizeof(struct mcx_eq_ctx) == 64);

struct mcx_cmd_create_eq_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_create_eq_mb_in {
	struct mcx_eq_ctx	cmd_eq_ctx;
	uint8_t			cmd_reserved0[8];
	uint64_t		cmd_event_bitmask;
#define MCX_EVENT_TYPE_COMPLETION	0x00
#define MCX_EVENT_TYPE_CQ_ERROR		0x04
#define MCX_EVENT_TYPE_INTERNAL_ERROR	0x08
#define MCX_EVENT_TYPE_PORT_CHANGE	0x09
#define MCX_EVENT_TYPE_CMD_COMPLETION	0x0a
#define MCX_EVENT_TYPE_PAGE_REQUEST	0x0b
#define MCX_EVENT_TYPE_LAST_WQE		0x13
	uint8_t			cmd_reserved1[176];
} __packed __aligned(4);

struct mcx_cmd_create_eq_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint32_t		cmd_eqn;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_cmd_query_eq_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint32_t		cmd_eqn;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_cmd_query_eq_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_eq_entry {
	uint8_t			eq_reserved1;
	uint8_t			eq_event_type;
	uint8_t			eq_reserved2;
	uint8_t			eq_event_sub_type;

	uint8_t			eq_reserved3[28];
	uint32_t		eq_event_data[7];
	uint8_t			eq_reserved4[2];
	uint8_t			eq_signature;
	uint8_t			eq_owner;
#define MCX_EQ_ENTRY_OWNER_INIT			1
} __packed __aligned(4);

CTASSERT(sizeof(struct mcx_eq_entry) == 64);

struct mcx_cmd_alloc_pd_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_alloc_pd_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint32_t		cmd_pd;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_cmd_alloc_td_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_alloc_td_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint32_t		cmd_tdomain;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_cmd_create_tir_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_create_tir_mb_in {
	uint8_t			cmd_reserved0[20];
	uint32_t		cmd_disp_type;
#define MCX_TIR_CTX_DISP_TYPE_DIRECT	0
#define MCX_TIR_CTX_DISP_TYPE_INDIRECT	1
#define MCX_TIR_CTX_DISP_TYPE_SHIFT	28
	uint8_t			cmd_reserved1[8];
	uint32_t		cmd_lro;
	uint8_t			cmd_reserved2[8];
	uint32_t		cmd_inline_rqn;
	uint32_t		cmd_indir_table;
	uint32_t		cmd_tdomain;
#define MCX_TIR_CTX_HASH_TOEPLITZ	2
#define MCX_TIR_CTX_HASH_SHIFT		28
	uint8_t			cmd_rx_hash_key[40];
	uint32_t		cmd_rx_hash_sel_outer;
#define MCX_TIR_CTX_HASH_SEL_SRC_IP	(1 << 0)
#define MCX_TIR_CTX_HASH_SEL_DST_IP	(1 << 1)
#define MCX_TIR_CTX_HASH_SEL_SPORT	(1 << 2)
#define MCX_TIR_CTX_HASH_SEL_DPORT	(1 << 3)
#define MCX_TIR_CTX_HASH_SEL_IPV4	(0 << 31)
#define MCX_TIR_CTX_HASH_SEL_IPV6	(1U << 31)
#define MCX_TIR_CTX_HASH_SEL_TCP	(0 << 30)
#define MCX_TIR_CTX_HASH_SEL_UDP	(1 << 30)
	uint32_t		cmd_rx_hash_sel_inner;
	uint8_t			cmd_reserved3[152];
} __packed __aligned(4);

struct mcx_cmd_create_tir_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint32_t		cmd_tirn;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_cmd_destroy_tir_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint32_t		cmd_tirn;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_cmd_destroy_tir_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_create_tis_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_create_tis_mb_in {
	uint8_t			cmd_reserved[16];
	uint32_t		cmd_prio;
	uint8_t			cmd_reserved1[32];
	uint32_t		cmd_tdomain;
	uint8_t			cmd_reserved2[120];
} __packed __aligned(4);

struct mcx_cmd_create_tis_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint32_t		cmd_tisn;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_cmd_destroy_tis_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint32_t		cmd_tisn;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_cmd_destroy_tis_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_create_rqt_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_rqt_ctx {
	uint8_t			cmd_reserved0[20];
	uint16_t		cmd_reserved1;
	uint16_t		cmd_rqt_max_size;
	uint16_t		cmd_reserved2;
	uint16_t		cmd_rqt_actual_size;
	uint8_t			cmd_reserved3[212];
} __packed __aligned(4);

struct mcx_cmd_create_rqt_mb_in {
	uint8_t			cmd_reserved0[16];
	struct mcx_rqt_ctx	cmd_rqt;
} __packed __aligned(4);

struct mcx_cmd_create_rqt_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint32_t		cmd_rqtn;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_cmd_destroy_rqt_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint32_t		cmd_rqtn;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_cmd_destroy_rqt_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cq_ctx {
	uint32_t		cq_status;
#define MCX_CQ_CTX_STATUS_SHIFT		28
#define MCX_CQ_CTX_STATUS_MASK		(0xf << MCX_CQ_CTX_STATUS_SHIFT)
#define MCX_CQ_CTX_STATUS_OK		0x0
#define MCX_CQ_CTX_STATUS_OVERFLOW	0x9
#define MCX_CQ_CTX_STATUS_WRITE_FAIL	0xa
#define MCX_CQ_CTX_STATE_SHIFT		8
#define MCX_CQ_CTX_STATE_MASK		(0xf << MCX_CQ_CTX_STATE_SHIFT)
#define MCX_CQ_CTX_STATE_SOLICITED	0x6
#define MCX_CQ_CTX_STATE_ARMED		0x9
#define MCX_CQ_CTX_STATE_FIRED		0xa
	uint32_t		cq_reserved1;
	uint32_t		cq_page_offset;
	uint32_t		cq_uar_size;
#define MCX_CQ_CTX_UAR_PAGE_MASK	0xffffff
#define MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT	24
	uint32_t		cq_period_max_count;
#define MCX_CQ_CTX_PERIOD_SHIFT		16
	uint32_t		cq_eqn;
	uint32_t		cq_log_page_size;
#define MCX_CQ_CTX_LOG_PAGE_SIZE_SHIFT	24
	uint32_t		cq_reserved2;
	uint32_t		cq_last_notified;
	uint32_t		cq_last_solicit;
	uint32_t		cq_consumer_counter;
	uint32_t		cq_producer_counter;
	uint8_t			cq_reserved3[8];
	uint64_t		cq_doorbell;
} __packed __aligned(4);

CTASSERT(sizeof(struct mcx_cq_ctx) == 64);

struct mcx_cmd_create_cq_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_create_cq_mb_in {
	struct mcx_cq_ctx	cmd_cq_ctx;
	uint8_t			cmd_reserved1[192];
} __packed __aligned(4);

struct mcx_cmd_create_cq_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint32_t		cmd_cqn;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_cmd_destroy_cq_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint32_t		cmd_cqn;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_cmd_destroy_cq_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_query_cq_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint32_t		cmd_cqn;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_cmd_query_cq_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cq_entry {
	uint32_t		__reserved__;
	uint32_t		cq_lro;
	uint32_t		cq_lro_ack_seq_num;
	uint32_t		cq_rx_hash;
	uint8_t			cq_rx_hash_type;
	uint8_t			cq_ml_path;
	uint16_t		__reserved__;
	uint32_t		cq_checksum;
	uint32_t		__reserved__;
	uint32_t		cq_flags;
#define MCX_CQ_ENTRY_FLAGS_L4_OK		(1 << 26)
#define MCX_CQ_ENTRY_FLAGS_L3_OK		(1 << 25)
#define MCX_CQ_ENTRY_FLAGS_L2_OK		(1 << 24)
#define MCX_CQ_ENTRY_FLAGS_CV			(1 << 16)
#define MCX_CQ_ENTRY_FLAGS_VLAN_MASK		(0xffff)

	uint32_t		cq_lro_srqn;
	uint32_t		__reserved__[2];
	uint32_t		cq_byte_cnt;
	uint64_t		cq_timestamp;
	uint8_t			cq_rx_drops;
	uint8_t			cq_flow_tag[3];
	uint16_t		cq_wqe_count;
	uint8_t			cq_signature;
	uint8_t			cq_opcode_owner;
#define MCX_CQ_ENTRY_FLAG_OWNER			(1 << 0)
#define MCX_CQ_ENTRY_FLAG_SE			(1 << 1)
#define MCX_CQ_ENTRY_FORMAT_SHIFT		2
#define MCX_CQ_ENTRY_OPCODE_SHIFT		4

#define MCX_CQ_ENTRY_FORMAT_NO_INLINE		0
#define MCX_CQ_ENTRY_FORMAT_INLINE_32		1
#define MCX_CQ_ENTRY_FORMAT_INLINE_64		2
#define MCX_CQ_ENTRY_FORMAT_COMPRESSED		3

#define MCX_CQ_ENTRY_OPCODE_REQ			0
#define MCX_CQ_ENTRY_OPCODE_SEND		2
#define MCX_CQ_ENTRY_OPCODE_REQ_ERR		13
#define MCX_CQ_ENTRY_OPCODE_SEND_ERR		14
#define MCX_CQ_ENTRY_OPCODE_INVALID		15

} __packed __aligned(4);

CTASSERT(sizeof(struct mcx_cq_entry) == 64);

struct mcx_cq_doorbell {
	uint32_t		 db_update_ci;
	uint32_t		 db_arm_ci;
#define MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT	28
#define MCX_CQ_DOORBELL_ARM_CMD			(1 << 24)
#define MCX_CQ_DOORBELL_ARM_CI_MASK		(0xffffff)
} __packed __aligned(8);

struct mcx_wq_ctx {
	uint8_t			 wq_type;
#define MCX_WQ_CTX_TYPE_CYCLIC			(1 << 4)
#define MCX_WQ_CTX_TYPE_SIGNATURE		(1 << 3)
	uint8_t			 wq_reserved0[5];
	uint16_t		 wq_lwm;
	uint32_t		 wq_pd;
	uint32_t		 wq_uar_page;
	uint64_t		 wq_doorbell;
	uint32_t		 wq_hw_counter;
	uint32_t		 wq_sw_counter;
	uint16_t		 wq_log_stride;
	uint8_t			 wq_log_page_sz;
	uint8_t			 wq_log_size;
	uint8_t			 wq_reserved1[156];
} __packed __aligned(4);

CTASSERT(sizeof(struct mcx_wq_ctx) == 0xC0);

struct mcx_sq_ctx {
	uint32_t		sq_flags;
#define MCX_SQ_CTX_RLKEY			(1U << 31)
#define MCX_SQ_CTX_FRE_SHIFT			(1 << 29)
#define MCX_SQ_CTX_FLUSH_IN_ERROR		(1 << 28)
#define MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT		24
#define MCX_SQ_CTX_STATE_SHIFT			20
#define MCX_SQ_CTX_STATE_MASK			(0xf << 20)
#define MCX_SQ_CTX_STATE_RST			0
#define MCX_SQ_CTX_STATE_RDY			1
#define MCX_SQ_CTX_STATE_ERR			3
	uint32_t		sq_user_index;
	uint32_t		sq_cqn;
	uint32_t		sq_reserved1[5];
	uint32_t		sq_tis_lst_sz;
#define MCX_SQ_CTX_TIS_LST_SZ_SHIFT		16
	uint32_t		sq_reserved2[2];
	uint32_t		sq_tis_num;
	struct mcx_wq_ctx	sq_wq;
} __packed __aligned(4);

struct mcx_sq_entry_seg {
	uint32_t		sqs_byte_count;
	uint32_t		sqs_lkey;
	uint64_t		sqs_addr;
} __packed __aligned(4);

struct mcx_sq_entry {
	/* control segment */
	uint32_t		sqe_opcode_index;
#define MCX_SQE_WQE_INDEX_SHIFT			8
#define MCX_SQE_WQE_OPCODE_NOP			0x00
#define MCX_SQE_WQE_OPCODE_SEND			0x0a
	uint32_t		sqe_ds_sq_num;
#define MCX_SQE_SQ_NUM_SHIFT			8
	uint32_t		sqe_signature;
#define MCX_SQE_SIGNATURE_SHIFT			24
#define MCX_SQE_SOLICITED_EVENT			0x02
#define MCX_SQE_CE_CQE_ON_ERR			0x00
#define MCX_SQE_CE_CQE_FIRST_ERR		0x04
#define MCX_SQE_CE_CQE_ALWAYS			0x08
#define MCX_SQE_CE_CQE_SOLICIT			0x0C
#define MCX_SQE_FM_NO_FENCE			0x00
#define MCX_SQE_FM_SMALL_FENCE			0x40
	uint32_t		sqe_mkey;

	/* ethernet segment */
	uint32_t		sqe_reserved1;
	uint32_t		sqe_mss_csum;
#define MCX_SQE_L4_CSUM				(1U << 31)
#define MCX_SQE_L3_CSUM				(1 << 30)
	uint32_t		sqe_reserved2;
	uint16_t		sqe_inline_header_size;
	uint16_t		sqe_inline_headers[9];

	/* data segment */
	struct mcx_sq_entry_seg sqe_segs[1];
} __packed __aligned(64);

CTASSERT(sizeof(struct mcx_sq_entry) == 64);

struct mcx_cmd_create_sq_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_create_sq_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint32_t		cmd_sqn;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_cmd_modify_sq_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint32_t		cmd_sq_state;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_cmd_modify_sq_mb_in {
	uint32_t		cmd_modify_hi;
	uint32_t		cmd_modify_lo;
	uint8_t			cmd_reserved0[8];
	struct mcx_sq_ctx	cmd_sq_ctx;
} __packed __aligned(4);

struct mcx_cmd_modify_sq_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_destroy_sq_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint32_t		cmd_sqn;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_cmd_destroy_sq_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);


struct mcx_rq_ctx {
	uint32_t		rq_flags;
#define MCX_RQ_CTX_RLKEY			(1U << 31)
#define MCX_RQ_CTX_VLAN_STRIP_DIS		(1 << 28)
#define MCX_RQ_CTX_MEM_RQ_TYPE_SHIFT		24
#define MCX_RQ_CTX_STATE_SHIFT			20
#define MCX_RQ_CTX_STATE_MASK			(0xf << 20)
#define MCX_RQ_CTX_STATE_RST			0
#define MCX_RQ_CTX_STATE_RDY			1
#define MCX_RQ_CTX_STATE_ERR			3
#define MCX_RQ_CTX_FLUSH_IN_ERROR		(1 << 18)
	uint32_t		rq_user_index;
	uint32_t		rq_cqn;
	uint32_t		rq_reserved1;
	uint32_t		rq_rmpn;
	uint32_t		rq_reserved2[7];
	struct mcx_wq_ctx	rq_wq;
} __packed __aligned(4);

struct mcx_rq_entry {
	uint32_t		rqe_byte_count;
	uint32_t		rqe_lkey;
	uint64_t		rqe_addr;
} __packed __aligned(16);

struct mcx_cmd_create_rq_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_create_rq_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint32_t		cmd_rqn;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_cmd_modify_rq_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint32_t		cmd_rq_state;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_cmd_modify_rq_mb_in {
	uint32_t		cmd_modify_hi;
	uint32_t		cmd_modify_lo;
	uint8_t			cmd_reserved0[8];
	struct mcx_rq_ctx	cmd_rq_ctx;
} __packed __aligned(4);

struct mcx_cmd_modify_rq_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_destroy_rq_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint32_t		cmd_rqn;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_cmd_destroy_rq_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_create_flow_table_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_flow_table_ctx {
	uint8_t			ft_miss_action;
	uint8_t			ft_level;
	uint8_t			ft_reserved0;
	uint8_t			ft_log_size;
	uint32_t		ft_table_miss_id;
	uint8_t			ft_reserved1[28];
} __packed __aligned(4);

struct mcx_cmd_create_flow_table_mb_in {
	uint8_t			cmd_table_type;
	uint8_t			cmd_reserved0[7];
	struct mcx_flow_table_ctx cmd_ctx;
} __packed __aligned(4);

struct mcx_cmd_create_flow_table_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint32_t		cmd_table_id;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_cmd_destroy_flow_table_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_destroy_flow_table_mb_in {
	uint8_t			cmd_table_type;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_table_id;
	uint8_t			cmd_reserved1[40];
} __packed __aligned(4);

struct mcx_cmd_destroy_flow_table_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_set_flow_table_root_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_set_flow_table_root_mb_in {
	uint8_t			cmd_table_type;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_table_id;
	uint8_t			cmd_reserved1[56];
} __packed __aligned(4);

struct mcx_cmd_set_flow_table_root_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_flow_match {
	/* outer headers */
	uint8_t			mc_src_mac[6];
	uint16_t		mc_ethertype;
	uint8_t			mc_dest_mac[6];
	uint16_t		mc_first_vlan;
	uint8_t			mc_ip_proto;
	uint8_t			mc_ip_dscp_ecn;
	uint8_t			mc_vlan_flags;
#define MCX_FLOW_MATCH_IP_FRAG	(1 << 5)
	uint8_t			mc_tcp_flags;
	uint16_t		mc_tcp_sport;
	uint16_t		mc_tcp_dport;
	uint32_t		mc_reserved0;
	uint16_t		mc_udp_sport;
	uint16_t		mc_udp_dport;
	uint8_t			mc_src_ip[16];
	uint8_t			mc_dest_ip[16];

	/* misc parameters */
	uint8_t			mc_reserved1[8];
	uint16_t		mc_second_vlan;
	uint8_t			mc_reserved2[2];
	uint8_t			mc_second_vlan_flags;
	uint8_t			mc_reserved3[15];
	uint32_t		mc_outer_ipv6_flow_label;
	uint8_t			mc_reserved4[32];

	uint8_t			mc_reserved[384];
} __packed __aligned(4);

CTASSERT(sizeof(struct mcx_flow_match) == 512);

struct mcx_cmd_create_flow_group_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_create_flow_group_mb_in {
	uint8_t			cmd_table_type;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_table_id;
	uint8_t			cmd_reserved1[4];
	uint32_t		cmd_start_flow_index;
	uint8_t			cmd_reserved2[4];
	uint32_t		cmd_end_flow_index;
	uint8_t			cmd_reserved3[23];
	uint8_t			cmd_match_criteria_enable;
#define MCX_CREATE_FLOW_GROUP_CRIT_OUTER	(1 << 0)
#define MCX_CREATE_FLOW_GROUP_CRIT_MISC		(1 << 1)
#define MCX_CREATE_FLOW_GROUP_CRIT_INNER	(1 << 2)
	struct mcx_flow_match	cmd_match_criteria;
	uint8_t			cmd_reserved4[448];
} __packed __aligned(4);

struct mcx_cmd_create_flow_group_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint32_t		cmd_group_id;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_flow_ctx {
	uint8_t			fc_reserved0[4];
	uint32_t		fc_group_id;
	uint32_t		fc_flow_tag;
	uint32_t		fc_action;
#define MCX_FLOW_CONTEXT_ACTION_ALLOW		(1 << 0)
#define MCX_FLOW_CONTEXT_ACTION_DROP		(1 << 1)
#define MCX_FLOW_CONTEXT_ACTION_FORWARD		(1 << 2)
#define MCX_FLOW_CONTEXT_ACTION_COUNT		(1 << 3)
	uint32_t		fc_dest_list_size;
	uint32_t		fc_counter_list_size;
	uint8_t			fc_reserved1[40];
	struct mcx_flow_match	fc_match_value;
	uint8_t			fc_reserved2[192];
} __packed __aligned(4);

#define MCX_FLOW_CONTEXT_DEST_TYPE_TABLE	(1 << 24)
#define MCX_FLOW_CONTEXT_DEST_TYPE_TIR		(2 << 24)

struct mcx_cmd_destroy_flow_group_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_destroy_flow_group_mb_in {
	uint8_t			cmd_table_type;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_table_id;
	uint32_t		cmd_group_id;
	uint8_t			cmd_reserved1[36];
} __packed __aligned(4);

struct mcx_cmd_destroy_flow_group_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_set_flow_table_entry_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_set_flow_table_entry_mb_in {
	uint8_t			cmd_table_type;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_table_id;
	uint32_t		cmd_modify_enable_mask;
	uint8_t			cmd_reserved1[4];
	uint32_t		cmd_flow_index;
	uint8_t			cmd_reserved2[28];
	struct mcx_flow_ctx	cmd_flow_ctx;
} __packed __aligned(4);

struct mcx_cmd_set_flow_table_entry_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_query_flow_table_entry_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_query_flow_table_entry_mb_in {
	uint8_t			cmd_table_type;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_table_id;
	uint8_t			cmd_reserved1[8];
	uint32_t		cmd_flow_index;
	uint8_t			cmd_reserved2[28];
} __packed __aligned(4);

struct mcx_cmd_query_flow_table_entry_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_query_flow_table_entry_mb_out {
	uint8_t			cmd_reserved0[48];
	struct mcx_flow_ctx	cmd_flow_ctx;
} __packed __aligned(4);

struct mcx_cmd_delete_flow_table_entry_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_delete_flow_table_entry_mb_in {
	uint8_t			cmd_table_type;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_table_id;
	uint8_t			cmd_reserved1[8];
	uint32_t		cmd_flow_index;
	uint8_t			cmd_reserved2[28];
} __packed __aligned(4);

struct mcx_cmd_delete_flow_table_entry_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_query_flow_group_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_query_flow_group_mb_in {
	uint8_t			cmd_table_type;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_table_id;
	uint32_t		cmd_group_id;
	uint8_t			cmd_reserved1[36];
} __packed __aligned(4);

struct mcx_cmd_query_flow_group_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_query_flow_group_mb_out {
	uint8_t			cmd_reserved0[12];
	uint32_t		cmd_start_flow_index;
	uint8_t			cmd_reserved1[4];
	uint32_t		cmd_end_flow_index;
	uint8_t			cmd_reserved2[20];
	uint32_t		cmd_match_criteria_enable;
	uint8_t			cmd_match_criteria[512];
	uint8_t			cmd_reserved4[448];
} __packed __aligned(4);

struct mcx_cmd_query_flow_table_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_query_flow_table_mb_in {
	uint8_t			cmd_table_type;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_table_id;
	uint8_t			cmd_reserved1[40];
} __packed __aligned(4);

struct mcx_cmd_query_flow_table_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_query_flow_table_mb_out {
	uint8_t			cmd_reserved0[4];
	struct mcx_flow_table_ctx cmd_ctx;
} __packed __aligned(4);

struct mcx_cmd_alloc_flow_counter_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_query_rq_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint32_t		cmd_rqn;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_cmd_query_rq_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_query_rq_mb_out {
	uint8_t			cmd_reserved0[16];
	struct mcx_rq_ctx	cmd_ctx;
};

struct mcx_cmd_query_sq_in {
	uint16_t		cmd_opcode;
	uint8_t			cmd_reserved0[4];
	uint16_t		cmd_op_mod;
	uint32_t		cmd_sqn;
	uint8_t			cmd_reserved1[4];
} __packed __aligned(4);

struct mcx_cmd_query_sq_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[8];
} __packed __aligned(4);

struct mcx_cmd_query_sq_mb_out {
	uint8_t			cmd_reserved0[16];
	struct mcx_sq_ctx	cmd_ctx;
};

struct mcx_cmd_alloc_flow_counter_out {
	uint8_t			cmd_status;
	uint8_t			cmd_reserved0[3];
	uint32_t		cmd_syndrome;
	uint8_t			cmd_reserved1[2];
	uint16_t		cmd_flow_counter_id;
	uint8_t			cmd_reserved2[4];
} __packed __aligned(4);

struct mcx_wq_doorbell {
	uint32_t		 db_recv_counter;
	uint32_t		 db_send_counter;
} __packed __aligned(8);

struct mcx_dmamem {
	bus_dmamap_t		 mxm_map;
	bus_dma_segment_t	 mxm_seg;
	int			 mxm_nsegs;
	size_t			 mxm_size;
	caddr_t			 mxm_kva;
};
#define MCX_DMA_MAP(_mxm)	((_mxm)->mxm_map)
#define MCX_DMA_DVA(_mxm)	((_mxm)->mxm_map->dm_segs[0].ds_addr)
#define MCX_DMA_KVA(_mxm)	((void *)(_mxm)->mxm_kva)
#define MCX_DMA_OFF(_mxm, _off)	((void *)((_mxm)->mxm_kva + (_off)))
#define MCX_DMA_LEN(_mxm)	((_mxm)->mxm_size)

struct mcx_hwmem {
	bus_dmamap_t		 mhm_map;
	bus_dma_segment_t	*mhm_segs;
	unsigned int		 mhm_seg_count;
	unsigned int		 mhm_npages;
};

struct mcx_slot {
	bus_dmamap_t		 ms_map;
	struct mbuf		*ms_m;
};

struct mcx_eq {
	int			 eq_n;
	uint32_t		 eq_cons;
	struct mcx_dmamem	 eq_mem;
};

struct mcx_cq {
	int			 cq_n;
	struct mcx_dmamem	 cq_mem;
	bus_addr_t		 cq_doorbell;
	uint32_t		 cq_cons;
	uint32_t		 cq_count;
};

struct mcx_calibration {
	uint64_t		 c_timestamp;	/* previous mcx chip time */
	uint64_t		 c_uptime;	/* previous kernel nanouptime */
	uint64_t		 c_tbase;	/* mcx chip time */
	uint64_t		 c_ubase;	/* kernel nanouptime */
	uint64_t		 c_ratio;
};

#define MCX_CALIBRATE_FIRST    2
#define MCX_CALIBRATE_NORMAL   32

struct mcx_rx {
	struct mcx_softc	*rx_softc;
	struct ifiqueue		*rx_ifiq;

	int			 rx_rqn;
	struct mcx_dmamem	 rx_rq_mem;
	struct mcx_slot		*rx_slots;
	bus_addr_t		 rx_doorbell;

	uint32_t		 rx_prod;
	struct timeout		 rx_refill;
	struct if_rxring	 rx_rxr;
} __aligned(64);

struct mcx_tx {
	struct mcx_softc	*tx_softc;
	struct ifqueue		*tx_ifq;

	int			 tx_uar;
	int			 tx_sqn;
	struct mcx_dmamem	 tx_sq_mem;
	struct mcx_slot		*tx_slots;
	bus_addr_t		 tx_doorbell;
	int			 tx_bf_offset;

	uint32_t		 tx_cons;
	uint32_t		 tx_prod;
} __aligned(64);

struct mcx_queues {
	char			 q_name[16];
	void			*q_ihc;
	struct mcx_softc	*q_sc;
	int			 q_uar;
	int			 q_index;
	struct mcx_rx		 q_rx;
	struct mcx_tx		 q_tx;
	struct mcx_cq		 q_cq;
	struct mcx_eq		 q_eq;
#if NKSTAT > 0
	struct kstat		*q_kstat;
#endif
};

struct mcx_flow_group {
	int			 g_id;
	int			 g_table;
	int			 g_start;
	int			 g_size;
};

#define MCX_FLOW_GROUP_PROMISC	 0
#define MCX_FLOW_GROUP_ALLMULTI	 1
#define MCX_FLOW_GROUP_MAC	 2
#define MCX_FLOW_GROUP_RSS_L4	 3
#define MCX_FLOW_GROUP_RSS_L3	 4
#define MCX_FLOW_GROUP_RSS_NONE	 5
#define MCX_NUM_FLOW_GROUPS	 6

#define MCX_HASH_SEL_L3		MCX_TIR_CTX_HASH_SEL_SRC_IP | \
				MCX_TIR_CTX_HASH_SEL_DST_IP
#define MCX_HASH_SEL_L4		MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_SPORT | \
				MCX_TIR_CTX_HASH_SEL_DPORT

#define MCX_RSS_HASH_SEL_V4_TCP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP  |\
				MCX_TIR_CTX_HASH_SEL_IPV4
#define MCX_RSS_HASH_SEL_V6_TCP	MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP | \
				MCX_TIR_CTX_HASH_SEL_IPV6
#define MCX_RSS_HASH_SEL_V4_UDP	MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \
				MCX_TIR_CTX_HASH_SEL_IPV4
#define MCX_RSS_HASH_SEL_V6_UDP	MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \
				MCX_TIR_CTX_HASH_SEL_IPV6
#define MCX_RSS_HASH_SEL_V4	MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV4
#define MCX_RSS_HASH_SEL_V6	MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV6

/*
 * There are a few different pieces involved in configuring RSS.
 * A Receive Queue Table (RQT) is the indirection table that maps packets to
 * different rx queues based on a hash value.  We only create one, because
 * we want to scatter any traffic we can apply RSS to across all our rx
 * queues.  Anything else will only be delivered to the first rx queue,
 * which doesn't require an RQT.
 *
 * A Transport Interface Receive (TIR) delivers packets to either a single rx
 * queue or an RQT, and in the latter case, specifies the set of fields
 * hashed, the hash function, and the hash key.  We need one of these for each
 * type of RSS traffic - v4 TCP, v6 TCP, v4 UDP, v6 UDP, other v4, other v6,
 * and one for non-RSS traffic.
 *
 * Flow tables hold flow table entries in sequence.  The first entry that
 * matches a packet is applied, sending the packet to either another flow
 * table or a TIR.  We use one flow table to select packets based on
 * destination MAC address, and a second to apply RSS.  The entries in the
 * first table send matching packets to the second, and the entries in the
 * RSS table send packets to RSS TIRs if possible, or the non-RSS TIR.
 *
 * The flow table entry that delivers packets to an RSS TIR must include match
 * criteria that ensure packets delivered to the TIR include all the fields
 * that the TIR hashes on - so for a v4 TCP TIR, the flow table entry must
 * only accept v4 TCP packets.  Accordingly, we need flow table entries for
 * each TIR.
 *
 * All of this is a lot more flexible than we need, and we can describe most
 * of the stuff we need with a simple array.
 *
 * An RSS config creates a TIR with hashing enabled on a set of fields,
 * pointing to either the first rx queue or the RQT containing all the rx
 * queues, and a flow table entry that matches on an ether type and
 * optionally an ip proto, that delivers packets to the TIR.
 */
static struct mcx_rss_rule {
	int			hash_sel;
	int			flow_group;
	int			ethertype;
	int			ip_proto;
} mcx_rss_config[] = {
	/* udp and tcp for v4/v6 */
	{ MCX_RSS_HASH_SEL_V4_TCP, MCX_FLOW_GROUP_RSS_L4,
	  ETHERTYPE_IP, IPPROTO_TCP },
	{ MCX_RSS_HASH_SEL_V6_TCP, MCX_FLOW_GROUP_RSS_L4,
	  ETHERTYPE_IPV6, IPPROTO_TCP },
	{ MCX_RSS_HASH_SEL_V4_UDP, MCX_FLOW_GROUP_RSS_L4,
	  ETHERTYPE_IP, IPPROTO_UDP },
	{ MCX_RSS_HASH_SEL_V6_UDP, MCX_FLOW_GROUP_RSS_L4,
	  ETHERTYPE_IPV6, IPPROTO_UDP },

	/* other v4/v6 */
	{ MCX_RSS_HASH_SEL_V4, MCX_FLOW_GROUP_RSS_L3,
	  ETHERTYPE_IP, 0 },
	{ MCX_RSS_HASH_SEL_V6, MCX_FLOW_GROUP_RSS_L3,
	  ETHERTYPE_IPV6, 0 },

	/* non v4/v6 */
	{ 0, MCX_FLOW_GROUP_RSS_NONE, 0, 0 }
};

struct mcx_softc {
	struct device		 sc_dev;
	struct arpcom		 sc_ac;
	struct ifmedia		 sc_media;
	uint64_t		 sc_media_status;
	uint64_t		 sc_media_active;

	pci_chipset_tag_t	 sc_pc;
	pci_intr_handle_t	 sc_ih;
	void			*sc_ihc;
	pcitag_t		 sc_tag;

	bus_dma_tag_t		 sc_dmat;
	bus_space_tag_t		 sc_memt;
	bus_space_handle_t	 sc_memh;
	bus_size_t		 sc_mems;

	struct mcx_dmamem	 sc_cmdq_mem;
	unsigned int		 sc_cmdq_mask;
	unsigned int		 sc_cmdq_size;

	unsigned int		 sc_cmdq_token;
	struct mutex		 sc_cmdq_mtx;
	struct rwlock		 sc_cmdq_kstat_lk;
	struct rwlock		 sc_cmdq_ioctl_lk;

	struct mcx_hwmem	 sc_boot_pages;
	struct mcx_hwmem	 sc_init_pages;
	struct mcx_hwmem	 sc_regular_pages;

	int			 sc_uar;
	int			 sc_pd;
	int			 sc_tdomain;
	uint32_t		 sc_lkey;
	int			 sc_tis;
	int			 sc_tir[nitems(mcx_rss_config)];
	int			 sc_rqt;

	struct mcx_dmamem	 sc_doorbell_mem;

	struct mcx_eq		 sc_admin_eq;
	struct mcx_eq		 sc_queue_eq;

	int			 sc_hardmtu;
	int			 sc_rxbufsz;

	int			 sc_bf_size;
	int			 sc_max_rqt_size;

	struct task		 sc_port_change;

	int			 sc_mac_flow_table_id;
	int			 sc_rss_flow_table_id;
	struct mcx_flow_group	 sc_flow_group[MCX_NUM_FLOW_GROUPS];
	int			 sc_promisc_flow_enabled;
	int			 sc_allmulti_flow_enabled;
	int			 sc_mcast_flow_base;
	int			 sc_extra_mcast;
	uint8_t			 sc_mcast_flows[MCX_NUM_MCAST_FLOWS][ETHER_ADDR_LEN];

	struct mcx_calibration	 sc_calibration[2];
	unsigned int		 sc_calibration_gen;
	struct timeout		 sc_calibrate;
	uint32_t		 sc_mhz;
	uint32_t		 sc_khz;

	struct intrmap		*sc_intrmap;
	struct mcx_queues	*sc_queues;

	int			 sc_mcam_reg;

#if NKSTAT > 0
	struct kstat		*sc_kstat_ieee8023;
	struct kstat		*sc_kstat_rfc2863;
	struct kstat		*sc_kstat_rfc2819;
	struct kstat		*sc_kstat_rfc3635;
	unsigned int		 sc_kstat_mtmp_count;
	struct kstat		**sc_kstat_mtmp;
#endif

	struct timecounter	 sc_timecounter;
};
#define DEVNAME(_sc) ((_sc)->sc_dev.dv_xname)

static int	mcx_match(struct device *, void *, void *);
static void	mcx_attach(struct device *, struct device *, void *);

#if NKSTAT > 0
static void	mcx_kstat_attach(struct mcx_softc *);
#endif

static void	mcx_timecounter_attach(struct mcx_softc *);

static int	mcx_version(struct mcx_softc *);
static int	mcx_init_wait(struct mcx_softc *);
static int	mcx_enable_hca(struct mcx_softc *);
static int	mcx_teardown_hca(struct mcx_softc *, uint16_t);
static int	mcx_access_hca_reg(struct mcx_softc *, uint16_t, int, void *,
		    int, enum mcx_cmdq_slot);
static int	mcx_issi(struct mcx_softc *);
static int	mcx_pages(struct mcx_softc *, struct mcx_hwmem *, uint16_t);
static int	mcx_hca_max_caps(struct mcx_softc *);
static int	mcx_hca_set_caps(struct mcx_softc *);
static int	mcx_init_hca(struct mcx_softc *);
static int	mcx_set_driver_version(struct mcx_softc *);
static int	mcx_iff(struct mcx_softc *);
static int	mcx_alloc_uar(struct mcx_softc *, int *);
static int	mcx_alloc_pd(struct mcx_softc *);
static int	mcx_alloc_tdomain(struct mcx_softc *);
static int	mcx_create_eq(struct mcx_softc *, struct mcx_eq *, int,
		    uint64_t, int);
static int	mcx_query_nic_vport_context(struct mcx_softc *);
static int	mcx_query_special_contexts(struct mcx_softc *);
static int	mcx_set_port_mtu(struct mcx_softc *, int);
static int	mcx_create_cq(struct mcx_softc *, struct mcx_cq *, int, int,
		    int);
static int	mcx_destroy_cq(struct mcx_softc *, struct mcx_cq *);
static int	mcx_create_sq(struct mcx_softc *, struct mcx_tx *, int, int,
		    int);
static int	mcx_destroy_sq(struct mcx_softc *, struct mcx_tx *);
static int	mcx_ready_sq(struct mcx_softc *, struct mcx_tx *);
static int	mcx_create_rq(struct mcx_softc *, struct mcx_rx *, int, int);
static int	mcx_destroy_rq(struct mcx_softc *, struct mcx_rx *);
static int	mcx_ready_rq(struct mcx_softc *, struct mcx_rx *);
static int	mcx_create_tir_direct(struct mcx_softc *, struct mcx_rx *,
		    int *);
static int	mcx_create_tir_indirect(struct mcx_softc *, int, uint32_t,
		    int *);
static int	mcx_destroy_tir(struct mcx_softc *, int);
static int	mcx_create_tis(struct mcx_softc *, int *);
static int	mcx_destroy_tis(struct mcx_softc *, int);
static int	mcx_create_rqt(struct mcx_softc *, int, int *, int *);
static int	mcx_destroy_rqt(struct mcx_softc *, int);
static int	mcx_create_flow_table(struct mcx_softc *, int, int, int *);
static int	mcx_set_flow_table_root(struct mcx_softc *, int);
static int	mcx_destroy_flow_table(struct mcx_softc *, int);
static int	mcx_create_flow_group(struct mcx_softc *, int, int, int,
		    int, int, struct mcx_flow_match *);
static int	mcx_destroy_flow_group(struct mcx_softc *, int);
static int	mcx_set_flow_table_entry_mac(struct mcx_softc *, int, int,
		    uint8_t *, uint32_t);
static int	mcx_set_flow_table_entry_proto(struct mcx_softc *, int, int,
		    int, int, uint32_t);
static int	mcx_delete_flow_table_entry(struct mcx_softc *, int, int);

#if NKSTAT > 0
static int	mcx_query_rq(struct mcx_softc *, struct mcx_rx *, struct mcx_rq_ctx *);
static int	mcx_query_sq(struct mcx_softc *, struct mcx_tx *, struct mcx_sq_ctx *);
static int	mcx_query_cq(struct mcx_softc *, struct mcx_cq *, struct mcx_cq_ctx *);
static int	mcx_query_eq(struct mcx_softc *, struct mcx_eq *, struct mcx_eq_ctx *);
#endif

#if 0
static int	mcx_dump_flow_table(struct mcx_softc *, int);
static int	mcx_dump_flow_table_entry(struct mcx_softc *, int, int);
static int	mcx_dump_flow_group(struct mcx_softc *, int);
#endif


/*
static void	mcx_cmdq_dump(const struct mcx_cmdq_entry *);
static void	mcx_cmdq_mbox_dump(struct mcx_dmamem *, int);
*/
static void	mcx_refill(void *);
static int	mcx_process_rx(struct mcx_softc *, struct mcx_rx *,
		    struct mcx_cq_entry *, struct mbuf_list *,
		    const struct mcx_calibration *);
static int	mcx_process_txeof(struct mcx_softc *, struct mcx_tx *,
		    struct mcx_cq_entry *);
static void	mcx_process_cq(struct mcx_softc *, struct mcx_queues *,
		    struct mcx_cq *);

static void	mcx_arm_cq(struct mcx_softc *, struct mcx_cq *, int);
static void	mcx_arm_eq(struct mcx_softc *, struct mcx_eq *, int);
static int	mcx_admin_intr(void *);
static int	mcx_cq_intr(void *);

static int	mcx_up(struct mcx_softc *);
static void	mcx_down(struct mcx_softc *);
static int	mcx_ioctl(struct ifnet *, u_long, caddr_t);
static int	mcx_rxrinfo(struct mcx_softc *, struct if_rxrinfo *);
static void	mcx_start(struct ifqueue *);
static void	mcx_watchdog(struct ifnet *);
static void	mcx_media_add_types(struct mcx_softc *);
static void	mcx_media_status(struct ifnet *, struct ifmediareq *);
static int	mcx_media_change(struct ifnet *);
static int	mcx_get_sffpage(struct ifnet *, struct if_sffpage *);
static void	mcx_port_change(void *);

static void	mcx_calibrate_first(struct mcx_softc *);
static void	mcx_calibrate(void *);

static inline uint32_t
		mcx_rd(struct mcx_softc *, bus_size_t);
static inline void
		mcx_wr(struct mcx_softc *, bus_size_t, uint32_t);
static inline void
		mcx_bar(struct mcx_softc *, bus_size_t, bus_size_t, int);

static uint64_t	mcx_timer(struct mcx_softc *);

static int	mcx_dmamem_alloc(struct mcx_softc *, struct mcx_dmamem *,
		    bus_size_t, u_int align);
static void	mcx_dmamem_zero(struct mcx_dmamem *);
static void	mcx_dmamem_free(struct mcx_softc *, struct mcx_dmamem *);

static int	mcx_hwmem_alloc(struct mcx_softc *, struct mcx_hwmem *,
		    unsigned int);
static void	mcx_hwmem_free(struct mcx_softc *, struct mcx_hwmem *);

struct cfdriver mcx_cd = {
	NULL,
	"mcx",
	DV_IFNET,
};

const struct cfattach mcx_ca = {
	sizeof(struct mcx_softc),
	mcx_match,
	mcx_attach,
};

static const struct pci_matchid mcx_devices[] = {
	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27700 },
	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27700VF },
	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27710 },
	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27710VF },
	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27800 },
	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27800VF },
	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT28800 },
	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT28800VF },
	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT28908 },
	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT28908VF },
	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT2892 },
	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT2894 },
};

struct mcx_eth_proto_capability {
	uint64_t	cap_media;
	uint64_t	cap_baudrate;
};

static const struct mcx_eth_proto_capability mcx_eth_cap_map[] = {
	[MCX_ETHER_CAP_SGMII]		= { IFM_1000_SGMII,	IF_Gbps(1) },
	[MCX_ETHER_CAP_1000_KX]		= { IFM_1000_KX,	IF_Gbps(1) },
	[MCX_ETHER_CAP_10G_CX4]		= { IFM_10G_CX4,	IF_Gbps(10) },
	[MCX_ETHER_CAP_10G_KX4]		= { IFM_10G_KX4,	IF_Gbps(10) },
	[MCX_ETHER_CAP_10G_KR]		= { IFM_10G_KR,		IF_Gbps(10) },
	[MCX_ETHER_CAP_40G_CR4]		= { IFM_40G_CR4,	IF_Gbps(40) },
	[MCX_ETHER_CAP_40G_KR4]		= { IFM_40G_KR4,	IF_Gbps(40) },
	[MCX_ETHER_CAP_10G_CR]		= { IFM_10G_SFP_CU,	IF_Gbps(10) },
	[MCX_ETHER_CAP_10G_SR]		= { IFM_10G_SR,		IF_Gbps(10) },
	[MCX_ETHER_CAP_10G_LR]		= { IFM_10G_LR,		IF_Gbps(10) },
	[MCX_ETHER_CAP_40G_SR4]		= { IFM_40G_SR4,	IF_Gbps(40) },
	[MCX_ETHER_CAP_40G_LR4]		= { IFM_40G_LR4,	IF_Gbps(40) },
	[MCX_ETHER_CAP_50G_SR2]		= { 0 /*IFM_50G_SR2*/,	IF_Gbps(50) },
	[MCX_ETHER_CAP_100G_CR4]	= { IFM_100G_CR4,	IF_Gbps(100) },
	[MCX_ETHER_CAP_100G_SR4]	= { IFM_100G_SR4,	IF_Gbps(100) },
	[MCX_ETHER_CAP_100G_KR4]	= { IFM_100G_KR4,	IF_Gbps(100) },
	[MCX_ETHER_CAP_100G_LR4]	= { IFM_100G_LR4,	IF_Gbps(100) },
	[MCX_ETHER_CAP_25G_CR]		= { IFM_25G_CR,		IF_Gbps(25) },
	[MCX_ETHER_CAP_25G_KR]		= { IFM_25G_KR,		IF_Gbps(25) },
	[MCX_ETHER_CAP_25G_SR]		= { IFM_25G_SR,		IF_Gbps(25) },
	[MCX_ETHER_CAP_50G_CR2]		= { IFM_50G_CR2,	IF_Gbps(50) },
	[MCX_ETHER_CAP_50G_KR2]		= { IFM_50G_KR2,	IF_Gbps(50) },
};

static const struct mcx_eth_proto_capability mcx_ext_eth_cap_map[] = {
	[MCX_ETHER_EXT_CAP_SGMII_100]	= { IFM_100_FX,		IF_Mbps(100) },
	[MCX_ETHER_EXT_CAP_1000_X]	= { IFM_1000_SX,	IF_Gbps(1) },
	[MCX_ETHER_EXT_CAP_5G_R]	= { IFM_5000_T,		IF_Gbps(5) },
	[MCX_ETHER_EXT_CAP_XAUI]	= { IFM_10G_SFI,	IF_Gbps(10) },
	[MCX_ETHER_EXT_CAP_XLAUI]	= { IFM_40G_XLPPI,	IF_Gbps(40) },
	[MCX_ETHER_EXT_CAP_25G_AUI1]	= { 0 /*IFM_25G_AUI*/,	IF_Gbps(25) },
	[MCX_ETHER_EXT_CAP_50G_AUI2]	= { 0 /*IFM_50G_AUI*/,	IF_Gbps(50) },
	[MCX_ETHER_EXT_CAP_50G_AUI1]	= { 0 /*IFM_50G_AUI*/,	IF_Gbps(50) },
	[MCX_ETHER_EXT_CAP_CAUI4]	= { 0 /*IFM_100G_AUI*/,	IF_Gbps(100) },
	[MCX_ETHER_EXT_CAP_100G_AUI2]	= { 0 /*IFM_100G_AUI*/,	IF_Gbps(100) },
	[MCX_ETHER_EXT_CAP_200G_AUI4]	= { 0 /*IFM_200G_AUI*/,	IF_Gbps(200) },
	[MCX_ETHER_EXT_CAP_400G_AUI8]	= { 0 /*IFM_400G_AUI*/,	IF_Gbps(400) },
};

static int
mcx_get_id(uint32_t val)
{
	return betoh32(val) & 0x00ffffff;
}

static int
mcx_match(struct device *parent, void *match, void *aux)
{
	return (pci_matchbyid(aux, mcx_devices, nitems(mcx_devices)));
}

void
mcx_attach(struct device *parent, struct device *self, void *aux)
{
	struct mcx_softc *sc = (struct mcx_softc *)self;
	struct ifnet *ifp = &sc->sc_ac.ac_if;
	struct pci_attach_args *pa = aux;
	pcireg_t memtype;
	uint32_t r;
	unsigned int cq_stride;
	unsigned int cq_size;
	const char *intrstr;
	int i, msix;

	sc->sc_pc = pa->pa_pc;
	sc->sc_tag = pa->pa_tag;
	sc->sc_dmat = pa->pa_dmat;

	/* Map the PCI memory space */
	memtype = pci_mapreg_type(sc->sc_pc, sc->sc_tag, MCX_HCA_BAR);
	if (pci_mapreg_map(pa, MCX_HCA_BAR, memtype,
	    BUS_SPACE_MAP_PREFETCHABLE, &sc->sc_memt, &sc->sc_memh,
	    NULL, &sc->sc_mems, 0)) {
		printf(": unable to map register memory\n");
		return;
	}

	if (mcx_version(sc) != 0) {
		/* error printed by mcx_version */
		goto unmap;
	}

	r = mcx_rd(sc, MCX_CMDQ_ADDR_LO);
	cq_stride = 1 << MCX_CMDQ_LOG_STRIDE(r); /* size of the entries */
	cq_size = 1 << MCX_CMDQ_LOG_SIZE(r); /* number of entries */
	if (cq_size > MCX_MAX_CQE) {
		printf(", command queue size overflow %u\n", cq_size);
		goto unmap;
	}
	if (cq_stride < sizeof(struct mcx_cmdq_entry)) {
		printf(", command queue entry size underflow %u\n", cq_stride);
		goto unmap;
	}
	if (cq_stride * cq_size > MCX_PAGE_SIZE) {
		printf(", command queue page overflow\n");
		goto unmap;
	}

	if (mcx_dmamem_alloc(sc, &sc->sc_doorbell_mem, MCX_DOORBELL_AREA_SIZE,
	    MCX_PAGE_SIZE) != 0) {
		printf(", unable to allocate doorbell memory\n");
		goto unmap;
	}

	if (mcx_dmamem_alloc(sc, &sc->sc_cmdq_mem, MCX_PAGE_SIZE,
	    MCX_PAGE_SIZE) != 0) {
		printf(", unable to allocate command queue\n");
		goto dbfree;
	}

	mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32);
	mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint32_t),
	    BUS_SPACE_BARRIER_WRITE);
	mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem));
	mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint32_t),
	    BUS_SPACE_BARRIER_WRITE);

	if (mcx_init_wait(sc) != 0) {
		printf(", timeout waiting for init\n");
		goto cqfree;
	}

	sc->sc_cmdq_mask = cq_size - 1;
	sc->sc_cmdq_size = cq_stride;
	rw_init(&sc->sc_cmdq_kstat_lk, "mcxkstat");
	rw_init(&sc->sc_cmdq_ioctl_lk, "mcxioctl");
	mtx_init(&sc->sc_cmdq_mtx, IPL_NET);

	if (mcx_enable_hca(sc) != 0) {
		/* error printed by mcx_enable_hca */
		goto cqfree;
	}

	if (mcx_issi(sc) != 0) {
		/* error printed by mcx_issi */
		goto teardown;
	}

	if (mcx_pages(sc, &sc->sc_boot_pages,
	    htobe16(MCX_CMD_QUERY_PAGES_BOOT)) != 0) {
		/* error printed by mcx_pages */
		goto teardown;
	}

	if (mcx_hca_max_caps(sc) != 0) {
		/* error printed by mcx_hca_max_caps */
		goto teardown;
	}

	if (mcx_hca_set_caps(sc) != 0) {
		/* error printed by mcx_hca_set_caps */
		goto teardown;
	}

	if (mcx_pages(sc, &sc->sc_init_pages,
	    htobe16(MCX_CMD_QUERY_PAGES_INIT)) != 0) {
		/* error printed by mcx_pages */
		goto teardown;
	}

	if (mcx_init_hca(sc) != 0) {
		/* error printed by mcx_init_hca */
		goto teardown;
	}

	if (mcx_pages(sc, &sc->sc_regular_pages,
	    htobe16(MCX_CMD_QUERY_PAGES_REGULAR)) != 0) {
		/* error printed by mcx_pages */
		goto teardown;
	}

	/* apparently not necessary? */
	if (mcx_set_driver_version(sc) != 0) {
		/* error printed by mcx_set_driver_version */
		goto teardown;
	}

	if (mcx_iff(sc) != 0) {	/* modify nic vport context */
		/* error printed by mcx_iff? */
		goto teardown;
	}

	if (mcx_alloc_uar(sc, &sc->sc_uar) != 0) {
		/* error printed by mcx_alloc_uar */
		goto teardown;
	}

	if (mcx_alloc_pd(sc) != 0) {
		/* error printed by mcx_alloc_pd */
		goto teardown;
	}

	if (mcx_alloc_tdomain(sc) != 0) {
		/* error printed by mcx_alloc_tdomain */
		goto teardown;
	}

	msix = pci_intr_msix_count(pa);
	if (msix < 2) {
		printf(": not enough msi-x vectors\n");
		goto teardown;
	}

	/*
	 * PRM makes no mention of msi interrupts, just legacy and msi-x.
	 * mellanox support tells me legacy interrupts are not supported,
	 * so we're stuck with just msi-x.
	 */
	if (pci_intr_map_msix(pa, 0, &sc->sc_ih) != 0) {
		printf(": unable to map interrupt\n");
		goto teardown;
	}
	intrstr = pci_intr_string(sc->sc_pc, sc->sc_ih);
	sc->sc_ihc = pci_intr_establish(sc->sc_pc, sc->sc_ih,
	    IPL_NET | IPL_MPSAFE, mcx_admin_intr, sc, DEVNAME(sc));
	if (sc->sc_ihc == NULL) {
		printf(": unable to establish interrupt");
		if (intrstr != NULL)
			printf(" at %s", intrstr);
		printf("\n");
		goto teardown;
	}

	if (mcx_create_eq(sc, &sc->sc_admin_eq, sc->sc_uar,
	    (1ull << MCX_EVENT_TYPE_INTERNAL_ERROR) |
	    (1ull << MCX_EVENT_TYPE_PORT_CHANGE) |
	    (1ull << MCX_EVENT_TYPE_CMD_COMPLETION) |
	    (1ull << MCX_EVENT_TYPE_PAGE_REQUEST), 0) != 0) {
		/* error printed by mcx_create_eq */
		goto teardown;
	}

	if (mcx_query_nic_vport_context(sc) != 0) {
		/* error printed by mcx_query_nic_vport_context */
		goto teardown;
	}

	if (mcx_query_special_contexts(sc) != 0) {
		/* error printed by mcx_query_special_contexts */
		goto teardown;
	}

	if (mcx_set_port_mtu(sc, MCX_HARDMTU) != 0) {
		/* error printed by mcx_set_port_mtu */
		goto teardown;
	}

	printf(", %s, address %s\n", intrstr,
	    ether_sprintf(sc->sc_ac.ac_enaddr));

	msix--; /* admin ops took one */
	sc->sc_intrmap = intrmap_create(&sc->sc_dev, msix, MCX_MAX_QUEUES,
	    INTRMAP_POWEROF2);
	if (sc->sc_intrmap == NULL) {
		printf("%s: unable to create interrupt map\n", DEVNAME(sc));
		goto teardown;
	}
	sc->sc_queues = mallocarray(intrmap_count(sc->sc_intrmap),
	    sizeof(*sc->sc_queues), M_DEVBUF, M_WAITOK|M_ZERO);
	if (sc->sc_queues == NULL) {
		printf("%s: unable to create queues\n", DEVNAME(sc));
		goto intrunmap;
	}

	strlcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ);
	ifp->if_softc = sc;
	ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX;
	ifp->if_xflags = IFXF_MPSAFE;
	ifp->if_ioctl = mcx_ioctl;
	ifp->if_qstart = mcx_start;
	ifp->if_watchdog = mcx_watchdog;
	ifp->if_hardmtu = sc->sc_hardmtu;
	ifp->if_capabilities = IFCAP_VLAN_MTU | IFCAP_CSUM_IPv4 |
	    IFCAP_CSUM_UDPv4 | IFCAP_CSUM_UDPv6 | IFCAP_CSUM_TCPv4 |
	    IFCAP_CSUM_TCPv6;
#if NVLAN > 0
	ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING;
#endif
	ifq_init_maxlen(&ifp->if_snd, 1024);

	ifmedia_init(&sc->sc_media, IFM_IMASK, mcx_media_change,
	    mcx_media_status);
	mcx_media_add_types(sc);
	ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL);
	ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO);

	if_attach(ifp);
	ether_ifattach(ifp);

	if_attach_iqueues(ifp, intrmap_count(sc->sc_intrmap));
	if_attach_queues(ifp, intrmap_count(sc->sc_intrmap));
	for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) {
		struct ifiqueue *ifiq = ifp->if_iqs[i];
		struct ifqueue *ifq = ifp->if_ifqs[i];
		struct mcx_queues *q = &sc->sc_queues[i];
		struct mcx_rx *rx = &q->q_rx;
		struct mcx_tx *tx = &q->q_tx;
		pci_intr_handle_t ih;
		int vec;

		vec = i + 1;
		q->q_sc = sc;
		q->q_index = i;

		if (mcx_alloc_uar(sc, &q->q_uar) != 0) {
			printf("%s: unable to alloc uar %d\n",
			    DEVNAME(sc), i);
			goto intrdisestablish;
		}

		if (mcx_create_eq(sc, &q->q_eq, q->q_uar, 0, vec) != 0) {
			printf("%s: unable to create event queue %d\n",
			    DEVNAME(sc), i);
			goto intrdisestablish;
		}

		rx->rx_softc = sc;
		rx->rx_ifiq = ifiq;
		timeout_set(&rx->rx_refill, mcx_refill, rx);
		ifiq->ifiq_softc = rx;

		tx->tx_softc = sc;
		tx->tx_ifq = ifq;
		ifq->ifq_softc = tx;

		if (pci_intr_map_msix(pa, vec, &ih) != 0) {
			printf("%s: unable to map queue interrupt %d\n",
			    DEVNAME(sc), i);
			goto intrdisestablish;
		}
		snprintf(q->q_name, sizeof(q->q_name), "%s:%d",
		    DEVNAME(sc), i);
		q->q_ihc = pci_intr_establish_cpu(sc->sc_pc, ih,
		    IPL_NET | IPL_MPSAFE, intrmap_cpu(sc->sc_intrmap, i),
		    mcx_cq_intr, q, q->q_name);
		if (q->q_ihc == NULL) {
			printf("%s: unable to establish interrupt %d\n",
			    DEVNAME(sc), i);
			goto intrdisestablish;
		}
	}

	timeout_set(&sc->sc_calibrate, mcx_calibrate, sc);

	task_set(&sc->sc_port_change, mcx_port_change, sc);
	mcx_port_change(sc);

	sc->sc_mac_flow_table_id = -1;
	sc->sc_rss_flow_table_id = -1;
	sc->sc_rqt = -1;
	for (i = 0; i < MCX_NUM_FLOW_GROUPS; i++) {
		struct mcx_flow_group *mfg = &sc->sc_flow_group[i];
		mfg->g_id = -1;
		mfg->g_table = -1;
		mfg->g_size = 0;
		mfg->g_start = 0;
	}
	sc->sc_extra_mcast = 0;
	memset(sc->sc_mcast_flows, 0, sizeof(sc->sc_mcast_flows));

#if NKSTAT > 0
	mcx_kstat_attach(sc);
#endif
	mcx_timecounter_attach(sc);
	return;

intrdisestablish:
	for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) {
		struct mcx_queues *q = &sc->sc_queues[i];
		if (q->q_ihc == NULL)
			continue;
		pci_intr_disestablish(sc->sc_pc, q->q_ihc);
		q->q_ihc = NULL;
	}
	free(sc->sc_queues, M_DEVBUF,
	    intrmap_count(sc->sc_intrmap) * sizeof(*sc->sc_queues));
intrunmap:
	intrmap_destroy(sc->sc_intrmap);
	sc->sc_intrmap = NULL;
teardown:
	mcx_teardown_hca(sc, htobe16(MCX_CMD_TEARDOWN_HCA_GRACEFUL));
	/* error printed by mcx_teardown_hca, and we're already unwinding */
cqfree:
	mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32);
	mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t),
	    BUS_SPACE_BARRIER_WRITE);
	mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem) |
	    MCX_CMDQ_INTERFACE_DISABLED);
	mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint64_t),
	    BUS_SPACE_BARRIER_WRITE);

	mcx_wr(sc, MCX_CMDQ_ADDR_HI, 0);
	mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t),
	    BUS_SPACE_BARRIER_WRITE);
	mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_CMDQ_INTERFACE_DISABLED);

	mcx_dmamem_free(sc, &sc->sc_cmdq_mem);
dbfree:
	mcx_dmamem_free(sc, &sc->sc_doorbell_mem);
unmap:
	bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems);
	sc->sc_mems = 0;
}

static int
mcx_version(struct mcx_softc *sc)
{
	uint32_t fw0, fw1;
	uint16_t cmdif;

	fw0 = mcx_rd(sc, MCX_FW_VER);
	fw1 = mcx_rd(sc, MCX_CMDIF_FW_SUBVER);

	printf(": FW %u.%u.%04u", MCX_FW_VER_MAJOR(fw0),
	    MCX_FW_VER_MINOR(fw0), MCX_FW_VER_SUBMINOR(fw1));

	cmdif = MCX_CMDIF(fw1);
	if (cmdif != MCX_CMD_IF_SUPPORTED) {
		printf(", unsupported command interface %u\n", cmdif);
		return (-1);
	}

	return (0);
}

static int
mcx_init_wait(struct mcx_softc *sc)
{
	unsigned int i;
	uint32_t r;

	for (i = 0; i < 2000; i++) {
		r = mcx_rd(sc, MCX_STATE);
		if ((r & MCX_STATE_MASK) == MCX_STATE_READY)
			return (0);

		delay(1000);
		mcx_bar(sc, MCX_STATE, sizeof(uint32_t),
		    BUS_SPACE_BARRIER_READ);
	}

	return (-1);
}

static uint8_t
mcx_cmdq_poll(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
    unsigned int msec)
{
	unsigned int i;

	for (i = 0; i < msec; i++) {
		bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem),
		    0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_POSTRW);

		if ((cqe->cq_status & MCX_CQ_STATUS_OWN_MASK) ==
		    MCX_CQ_STATUS_OWN_SW)
			return (0);

		delay(1000);
	}

	return (ETIMEDOUT);
}

static uint32_t
mcx_mix_u64(uint32_t xor, uint64_t u64)
{
	xor ^= u64 >> 32;
	xor ^= u64;

	return (xor);
}

static uint32_t
mcx_mix_u32(uint32_t xor, uint32_t u32)
{
	xor ^= u32;

	return (xor);
}

static uint32_t
mcx_mix_u8(uint32_t xor, uint8_t u8)
{
	xor ^= u8;

	return (xor);
}

static uint8_t
mcx_mix_done(uint32_t xor)
{
	xor ^= xor >> 16;
	xor ^= xor >> 8;

	return (xor);
}

static uint8_t
mcx_xor(const void *buf, size_t len)
{
	const uint32_t *dwords = buf;
	uint32_t xor = 0xff;
	size_t i;

	len /= sizeof(*dwords);

	for (i = 0; i < len; i++)
		xor ^= dwords[i];

	return (mcx_mix_done(xor));
}

static uint8_t
mcx_cmdq_token(struct mcx_softc *sc)
{
	uint8_t token;

	mtx_enter(&sc->sc_cmdq_mtx);
	do {
		token = ++sc->sc_cmdq_token;
	} while (token == 0);
	mtx_leave(&sc->sc_cmdq_mtx);

	return (token);
}

static struct mcx_cmdq_entry *
mcx_get_cmdq_entry(struct mcx_softc *sc, enum mcx_cmdq_slot slot)
{
	struct mcx_cmdq_entry *cqe;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	cqe += slot;

	/* make sure the slot isn't running a command already */
	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem),
	    0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_POSTRW);
	if ((cqe->cq_status & MCX_CQ_STATUS_OWN_MASK) !=
	    MCX_CQ_STATUS_OWN_SW)
		cqe = NULL;

	return (cqe);
}

static void
mcx_cmdq_init(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
    uint32_t ilen, uint32_t olen, uint8_t token)
{
	memset(cqe, 0, sc->sc_cmdq_size);

	cqe->cq_type = MCX_CMDQ_TYPE_PCIE;
	htobem32(&cqe->cq_input_length, ilen);
	htobem32(&cqe->cq_output_length, olen);
	cqe->cq_token = token;
	cqe->cq_status = MCX_CQ_STATUS_OWN_HW;
}

static void
mcx_cmdq_sign(struct mcx_cmdq_entry *cqe)
{
	cqe->cq_signature = ~mcx_xor(cqe, sizeof(*cqe));
}

static int
mcx_cmdq_verify(const struct mcx_cmdq_entry *cqe)
{
	/* return (mcx_xor(cqe, sizeof(*cqe)) ? -1 :  0); */
	return (0);
}

static void *
mcx_cmdq_in(struct mcx_cmdq_entry *cqe)
{
	return (&cqe->cq_input_data);
}

static void *
mcx_cmdq_out(struct mcx_cmdq_entry *cqe)
{
	return (&cqe->cq_output_data);
}

static void
mcx_cmdq_post(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
    unsigned int slot)
{
	mcx_cmdq_sign(cqe);

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem),
	    0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_PRERW);

	mcx_wr(sc, MCX_CMDQ_DOORBELL, 1U << slot);
	mcx_bar(sc, MCX_CMDQ_DOORBELL, sizeof(uint32_t),
	    BUS_SPACE_BARRIER_WRITE);
}

static int
mcx_cmdq_exec(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
    unsigned int slot, unsigned int msec)
{
	int err;

	if (slot == MCX_CMDQ_SLOT_POLL) {
		mcx_cmdq_post(sc, cqe, slot);
		return (mcx_cmdq_poll(sc, cqe, msec));
	}

	mtx_enter(&sc->sc_cmdq_mtx);
	mcx_cmdq_post(sc, cqe, slot);

	err = 0;
	while (err == 0) {
		err = msleep_nsec(&sc->sc_cmdq_token, &sc->sc_cmdq_mtx, 0,
		    "mcxcmd", msec * 1000);
		bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem), 0,
		    MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_POSTRW);
		if ((cqe->cq_status & MCX_CQ_STATUS_OWN_MASK) ==
		    MCX_CQ_STATUS_OWN_SW) {
			err = 0;
			break;
		}
	}

	mtx_leave(&sc->sc_cmdq_mtx);
	return (err);
}

static int
mcx_enable_hca(struct mcx_softc *sc)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_enable_hca_in *in;
	struct mcx_cmd_enable_hca_out *out;
	int error;
	uint8_t status;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_ENABLE_HCA);
	in->cmd_op_mod = htobe16(0);
	in->cmd_function_id = htobe16(0);

	mcx_cmdq_post(sc, cqe, 0);

	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf(", hca enable timeout\n");
		return (-1);
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf(", hca enable command corrupt\n");
		return (-1);
	}

	status = cqe->cq_output_data[0];
	if (status != MCX_CQ_STATUS_OK) {
		printf(", hca enable failed (%x)\n", status);
		return (-1);
	}

	return (0);
}

static int
mcx_teardown_hca(struct mcx_softc *sc, uint16_t profile)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_teardown_hca_in *in;
	struct mcx_cmd_teardown_hca_out *out;
	int error;
	uint8_t status;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_TEARDOWN_HCA);
	in->cmd_op_mod = htobe16(0);
	in->cmd_profile = profile;

	mcx_cmdq_post(sc, cqe, 0);

	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf(", hca teardown timeout\n");
		return (-1);
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf(", hca teardown command corrupt\n");
		return (-1);
	}

	status = cqe->cq_output_data[0];
	if (status != MCX_CQ_STATUS_OK) {
		printf(", hca teardown failed (%x)\n", status);
		return (-1);
	}

	return (0);
}

static int
mcx_cmdq_mboxes_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm,
    unsigned int nmb, uint64_t *ptr, uint8_t token)
{
	caddr_t kva;
	uint64_t dva;
	int i;
	int error;

	error = mcx_dmamem_alloc(sc, mxm,
	    nmb * MCX_CMDQ_MAILBOX_SIZE, MCX_CMDQ_MAILBOX_ALIGN);
	if (error != 0)
		return (error);

	mcx_dmamem_zero(mxm);

	dva = MCX_DMA_DVA(mxm);
	kva = MCX_DMA_KVA(mxm);
	for (i = 0; i < nmb; i++) {
		struct mcx_cmdq_mailbox *mbox = (struct mcx_cmdq_mailbox *)kva;

		/* patch the cqe or mbox pointing at this one */
		htobem64(ptr, dva);

		/* fill in this mbox */
		htobem32(&mbox->mb_block_number, i);
		mbox->mb_token = token;

		/* move to the next one */
		ptr = &mbox->mb_next_ptr;

		dva += MCX_CMDQ_MAILBOX_SIZE;
		kva += MCX_CMDQ_MAILBOX_SIZE;
	}

	return (0);
}

static uint32_t
mcx_cmdq_mbox_ctrl_sig(const struct mcx_cmdq_mailbox *mb)
{
	uint32_t xor = 0xff;

	/* only 3 fields get set, so mix them directly */
	xor = mcx_mix_u64(xor, mb->mb_next_ptr);
	xor = mcx_mix_u32(xor, mb->mb_block_number);
	xor = mcx_mix_u8(xor, mb->mb_token);

	return (mcx_mix_done(xor));
}

static void
mcx_cmdq_mboxes_sign(struct mcx_dmamem *mxm, unsigned int nmb)
{
	caddr_t kva;
	int i;

	kva = MCX_DMA_KVA(mxm);

	for (i = 0; i < nmb; i++) {
		struct mcx_cmdq_mailbox *mb = (struct mcx_cmdq_mailbox *)kva;
		uint8_t sig = mcx_cmdq_mbox_ctrl_sig(mb);
		mb->mb_ctrl_signature = sig;
		mb->mb_signature = sig ^
		    mcx_xor(mb->mb_data, sizeof(mb->mb_data));

		kva += MCX_CMDQ_MAILBOX_SIZE;
	}
}

static void
mcx_cmdq_mboxes_sync(struct mcx_softc *sc, struct mcx_dmamem *mxm, int ops)
{
	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(mxm),
	    0, MCX_DMA_LEN(mxm), ops);
}

static struct mcx_cmdq_mailbox *
mcx_cq_mbox(struct mcx_dmamem *mxm, unsigned int i)
{
	caddr_t kva;

	kva = MCX_DMA_KVA(mxm);
	kva += i * MCX_CMDQ_MAILBOX_SIZE;

	return ((struct mcx_cmdq_mailbox *)kva);
}

static inline void *
mcx_cq_mbox_data(struct mcx_cmdq_mailbox *mb)
{
	return (&mb->mb_data);
}

static void
mcx_cmdq_mboxes_copyin(struct mcx_dmamem *mxm, unsigned int nmb,
    void *b, size_t len)
{
	caddr_t buf = b;
	struct mcx_cmdq_mailbox *mb;
	int i;

	mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm);
	for (i = 0; i < nmb; i++) {

		memcpy(mb->mb_data, buf, min(sizeof(mb->mb_data), len));

		if (sizeof(mb->mb_data) >= len)
			break;

		buf += sizeof(mb->mb_data);
		len -= sizeof(mb->mb_data);
		mb++;
	}
}

static void
mcx_cmdq_mboxes_pas(struct mcx_dmamem *mxm, int offset, int npages,
    struct mcx_dmamem *buf)
{
	uint64_t *pas;
	int mbox, mbox_pages, i;

	mbox = offset / MCX_CMDQ_MAILBOX_DATASIZE;
	offset %= MCX_CMDQ_MAILBOX_DATASIZE;

	pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox));
	pas += (offset / sizeof(*pas));
	mbox_pages = (MCX_CMDQ_MAILBOX_DATASIZE - offset) / sizeof(*pas);
	for (i = 0; i < npages; i++) {
		if (i == mbox_pages) {
			mbox++;
			pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox));
			mbox_pages += MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas);
		}
		*pas = htobe64(MCX_DMA_DVA(buf) + (i * MCX_PAGE_SIZE));
		pas++;
	}
}

static void
mcx_cmdq_mboxes_copyout(struct mcx_dmamem *mxm, int nmb, void *b, size_t len)
{
	caddr_t buf = b;
	struct mcx_cmdq_mailbox *mb;
	int i;

	mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm);
	for (i = 0; i < nmb; i++) {
		memcpy(buf, mb->mb_data, min(sizeof(mb->mb_data), len));

		if (sizeof(mb->mb_data) >= len)
			break;

		buf += sizeof(mb->mb_data);
		len -= sizeof(mb->mb_data);
		mb++;
	}
}

static void
mcx_cq_mboxes_free(struct mcx_softc *sc, struct mcx_dmamem *mxm)
{
	mcx_dmamem_free(sc, mxm);
}

#if 0
static void
mcx_cmdq_dump(const struct mcx_cmdq_entry *cqe)
{
	unsigned int i;

	printf(" type %02x, ilen %u, iptr %016llx", cqe->cq_type,
	    bemtoh32(&cqe->cq_input_length), bemtoh64(&cqe->cq_input_ptr));

	printf(", idata ");
	for (i = 0; i < sizeof(cqe->cq_input_data); i++)
		printf("%02x", cqe->cq_input_data[i]);

	printf(", odata ");
	for (i = 0; i < sizeof(cqe->cq_output_data); i++)
		printf("%02x", cqe->cq_output_data[i]);

	printf(", optr %016llx, olen %u, token %02x, sig %02x, status %02x",
	    bemtoh64(&cqe->cq_output_ptr), bemtoh32(&cqe->cq_output_length),
	    cqe->cq_token, cqe->cq_signature, cqe->cq_status);
}

static void
mcx_cmdq_mbox_dump(struct mcx_dmamem *mboxes, int num)
{
	int i, j;
	uint8_t *d;

	for (i = 0; i < num; i++) {
		struct mcx_cmdq_mailbox *mbox;
		mbox = mcx_cq_mbox(mboxes, i);

		d = mcx_cq_mbox_data(mbox);
		for (j = 0; j < MCX_CMDQ_MAILBOX_DATASIZE; j++) {
			if (j != 0 && (j % 16 == 0))
				printf("\n");
			printf("%.2x ", d[j]);
		}
	}
}
#endif

static int
mcx_access_hca_reg(struct mcx_softc *sc, uint16_t reg, int op, void *data,
    int len, enum mcx_cmdq_slot slot)
{
	struct mcx_dmamem mxm;
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_access_reg_in *in;
	struct mcx_cmd_access_reg_out *out;
	uint8_t token = mcx_cmdq_token(sc);
	int error, nmb;

	cqe = mcx_get_cmdq_entry(sc, slot);
	if (cqe == NULL)
		return (-1);

	mcx_cmdq_init(sc, cqe, sizeof(*in) + len, sizeof(*out) + len,
	    token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_ACCESS_REG);
	in->cmd_op_mod = htobe16(op);
	in->cmd_register_id = htobe16(reg);

	nmb = howmany(len, MCX_CMDQ_MAILBOX_DATASIZE);
	if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb,
	    &cqe->cq_output_ptr, token) != 0) {
		printf(", unable to allocate access reg mailboxen\n");
		return (-1);
	}
	cqe->cq_input_ptr = cqe->cq_output_ptr;
	mcx_cmdq_mboxes_copyin(&mxm, nmb, data, len);
	mcx_cmdq_mboxes_sign(&mxm, nmb);
	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);

	error = mcx_cmdq_exec(sc, cqe, slot, 1000);
	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);

	if (error != 0) {
		printf("%s: access reg (%s %x) timeout\n", DEVNAME(sc),
		    (op == MCX_REG_OP_WRITE ? "write" : "read"), reg);
		goto free;
	}
	error = mcx_cmdq_verify(cqe);
	if (error != 0) {
		printf("%s: access reg (%s %x) reply corrupt\n",
		    (op == MCX_REG_OP_WRITE ? "write" : "read"), DEVNAME(sc),
		    reg);
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: access reg (%s %x) failed (%x, %.6x)\n",
		    DEVNAME(sc), (op == MCX_REG_OP_WRITE ? "write" : "read"),
		    reg, out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

	mcx_cmdq_mboxes_copyout(&mxm, nmb, data, len);
free:
	mcx_dmamem_free(sc, &mxm);

	return (error);
}

static int
mcx_set_issi(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
    unsigned int slot)
{
	struct mcx_cmd_set_issi_in *in;
	struct mcx_cmd_set_issi_out *out;
	uint8_t status;

	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_SET_ISSI);
	in->cmd_op_mod = htobe16(0);
	in->cmd_current_issi = htobe16(MCX_ISSI);

	mcx_cmdq_post(sc, cqe, slot);
	if (mcx_cmdq_poll(sc, cqe, 1000) != 0)
		return (-1);
	if (mcx_cmdq_verify(cqe) != 0)
		return (-1);

	status = cqe->cq_output_data[0];
	if (status != MCX_CQ_STATUS_OK)
		return (-1);

	return (0);
}

static int
mcx_issi(struct mcx_softc *sc)
{
	struct mcx_dmamem mxm;
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_query_issi_in *in;
	struct mcx_cmd_query_issi_il_out *out;
	struct mcx_cmd_query_issi_mb_out *mb;
	uint8_t token = mcx_cmdq_token(sc);
	uint8_t status;
	int error;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mb), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_QUERY_ISSI);
	in->cmd_op_mod = htobe16(0);

	CTASSERT(sizeof(*mb) <= MCX_CMDQ_MAILBOX_DATASIZE);
	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
	    &cqe->cq_output_ptr, token) != 0) {
		printf(", unable to allocate query issi mailbox\n");
		return (-1);
	}
	mcx_cmdq_mboxes_sign(&mxm, 1);

	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf(", query issi timeout\n");
		goto free;
	}
	error = mcx_cmdq_verify(cqe);
	if (error != 0) {
		printf(", query issi reply corrupt\n");
		goto free;
	}

	status = cqe->cq_output_data[0];
	switch (status) {
	case MCX_CQ_STATUS_OK:
		break;
	case MCX_CQ_STATUS_BAD_OPCODE:
		/* use ISSI 0 */
		goto free;
	default:
		printf(", query issi failed (%x)\n", status);
		error = -1;
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_current_issi == htobe16(MCX_ISSI)) {
		/* use ISSI 1 */
		goto free;
	}

	/* don't need to read cqe anymore, can be used for SET ISSI */

	mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
	CTASSERT(MCX_ISSI < NBBY);
	 /* XXX math is hard */
	if (!ISSET(mb->cmd_supported_issi[79], 1 << MCX_ISSI)) {
		/* use ISSI 0 */
		goto free;
	}

	if (mcx_set_issi(sc, cqe, 0) != 0) {
		/* ignore the error, just use ISSI 0 */
	} else {
		/* use ISSI 1 */
	}

free:
	mcx_cq_mboxes_free(sc, &mxm);
	return (error);
}

static int
mcx_query_pages(struct mcx_softc *sc, uint16_t type,
    int32_t *npages, uint16_t *func_id)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_query_pages_in *in;
	struct mcx_cmd_query_pages_out *out;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_QUERY_PAGES);
	in->cmd_op_mod = type;

	mcx_cmdq_post(sc, cqe, 0);
	if (mcx_cmdq_poll(sc, cqe, 1000) != 0) {
		printf(", query pages timeout\n");
		return (-1);
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf(", query pages reply corrupt\n");
		return (-1);
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf(", query pages failed (%x)\n", out->cmd_status);
		return (-1);
	}

	*func_id = out->cmd_func_id;
	*npages = bemtoh32(&out->cmd_num_pages);

	return (0);
}

struct bus_dma_iter {
	bus_dmamap_t		i_map;
	bus_size_t		i_offset;
	unsigned int		i_index;
};

static void
bus_dma_iter_init(struct bus_dma_iter *i, bus_dmamap_t map)
{
	i->i_map = map;
	i->i_offset = 0;
	i->i_index = 0;
}

static bus_addr_t
bus_dma_iter_addr(struct bus_dma_iter *i)
{
	return (i->i_map->dm_segs[i->i_index].ds_addr + i->i_offset);
}

static void
bus_dma_iter_add(struct bus_dma_iter *i, bus_size_t size)
{
	bus_dma_segment_t *seg = i->i_map->dm_segs + i->i_index;
	bus_size_t diff;

	do {
		diff = seg->ds_len - i->i_offset;
		if (size < diff)
			break;

		size -= diff;

		seg++;

		i->i_offset = 0;
		i->i_index++;
	} while (size > 0);

	i->i_offset += size;
}

static int
mcx_add_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t func_id)
{
	struct mcx_dmamem mxm;
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_manage_pages_in *in;
	struct mcx_cmd_manage_pages_out *out;
	unsigned int paslen, nmb, i, j, npages;
	struct bus_dma_iter iter;
	uint64_t *pas;
	uint8_t status;
	uint8_t token = mcx_cmdq_token(sc);
	int error;

	npages = mhm->mhm_npages;

	paslen = sizeof(*pas) * npages;
	nmb = howmany(paslen, MCX_CMDQ_MAILBOX_DATASIZE);

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	mcx_cmdq_init(sc, cqe, sizeof(*in) + paslen, sizeof(*out), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_MANAGE_PAGES);
	in->cmd_op_mod = htobe16(MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS);
	in->cmd_func_id = func_id;
	htobem32(&in->cmd_input_num_entries, npages);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb,
	    &cqe->cq_input_ptr, token) != 0) {
		printf(", unable to allocate manage pages mailboxen\n");
		return (-1);
	}

	bus_dma_iter_init(&iter, mhm->mhm_map);
	for (i = 0; i < nmb; i++) {
		unsigned int lim;

		pas = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, i));
		lim = min(MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas), npages);

		for (j = 0; j < lim; j++) {
			htobem64(&pas[j], bus_dma_iter_addr(&iter));
			bus_dma_iter_add(&iter, MCX_PAGE_SIZE);
		}

		npages -= lim;
	}

	mcx_cmdq_mboxes_sign(&mxm, nmb);

	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf(", manage pages timeout\n");
		goto free;
	}
	error = mcx_cmdq_verify(cqe);
	if (error != 0) {
		printf(", manage pages reply corrupt\n");
		goto free;
	}

	status = cqe->cq_output_data[0];
	if (status != MCX_CQ_STATUS_OK) {
		printf(", manage pages failed (%x)\n", status);
		error = -1;
		goto free;
	}

free:
	mcx_dmamem_free(sc, &mxm);

	return (error);
}

static int
mcx_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t type)
{
	int32_t npages;
	uint16_t func_id;

	if (mcx_query_pages(sc, type, &npages, &func_id) != 0) {
		/* error printed by mcx_query_pages */
		return (-1);
	}

	if (npages < 1)
		return (0);

	if (mcx_hwmem_alloc(sc, mhm, npages) != 0) {
		printf(", unable to allocate hwmem\n");
		return (-1);
	}

	if (mcx_add_pages(sc, mhm, func_id) != 0) {
		printf(", unable to add hwmem\n");
		goto free;
	}

	return (0);

free:
	mcx_hwmem_free(sc, mhm);

	return (-1);
}

static int
mcx_hca_max_caps(struct mcx_softc *sc)
{
	struct mcx_dmamem mxm;
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_query_hca_cap_in *in;
	struct mcx_cmd_query_hca_cap_out *out;
	struct mcx_cmdq_mailbox *mb;
	struct mcx_cap_device *hca;
	uint8_t status;
	uint8_t token = mcx_cmdq_token(sc);
	int error;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN,
	    token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP);
	in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_MAX |
	    MCX_CMD_QUERY_HCA_CAP_DEVICE);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES,
	    &cqe->cq_output_ptr, token) != 0) {
		printf(", unable to allocate query hca caps mailboxen\n");
		return (-1);
	}
	mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES);
	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);

	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);

	if (error != 0) {
		printf(", query hca caps timeout\n");
		goto free;
	}
	error = mcx_cmdq_verify(cqe);
	if (error != 0) {
		printf(", query hca caps reply corrupt\n");
		goto free;
	}

	status = cqe->cq_output_data[0];
	if (status != MCX_CQ_STATUS_OK) {
		printf(", query hca caps failed (%x)\n", status);
		error = -1;
		goto free;
	}

	mb = mcx_cq_mbox(&mxm, 0);
	hca = mcx_cq_mbox_data(mb);

	if ((hca->port_type & MCX_CAP_DEVICE_PORT_TYPE)
	    != MCX_CAP_DEVICE_PORT_TYPE_ETH) {
		printf(", not in ethernet mode\n");
		error = -1;
		goto free;
	}
	if (hca->log_pg_sz > PAGE_SHIFT) {
		printf(", minimum system page shift %u is too large\n",
		    hca->log_pg_sz);
		error = -1;
		goto free;
	}
	/*
	 * blueflame register is split into two buffers, and we must alternate
	 * between the two of them.
	 */
	sc->sc_bf_size = (1 << hca->log_bf_reg_size) / 2;
	sc->sc_max_rqt_size = (1 << hca->log_max_rqt_size);
	
	if (hca->local_ca_ack_delay & MCX_CAP_DEVICE_MCAM_REG)
		sc->sc_mcam_reg = 1;

	sc->sc_mhz = bemtoh32(&hca->device_frequency_mhz);
	sc->sc_khz = bemtoh32(&hca->device_frequency_khz);

free:
	mcx_dmamem_free(sc, &mxm);

	return (error);
}

static int
mcx_hca_set_caps(struct mcx_softc *sc)
{
	struct mcx_dmamem mxm;
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_query_hca_cap_in *in;
	struct mcx_cmd_query_hca_cap_out *out;
	struct mcx_cmdq_mailbox *mb;
	struct mcx_cap_device *hca;
	uint8_t status;
	uint8_t token = mcx_cmdq_token(sc);
	int error;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN,
	    token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP);
	in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_CURRENT |
	    MCX_CMD_QUERY_HCA_CAP_DEVICE);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES,
	    &cqe->cq_output_ptr, token) != 0) {
		printf(", unable to allocate manage pages mailboxen\n");
		return (-1);
	}
	mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES);
	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);

	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);

	if (error != 0) {
		printf(", query hca caps timeout\n");
		goto free;
	}
	error = mcx_cmdq_verify(cqe);
	if (error != 0) {
		printf(", query hca caps reply corrupt\n");
		goto free;
	}

	status = cqe->cq_output_data[0];
	if (status != MCX_CQ_STATUS_OK) {
		printf(", query hca caps failed (%x)\n", status);
		error = -1;
		goto free;
	}

	mb = mcx_cq_mbox(&mxm, 0);
	hca = mcx_cq_mbox_data(mb);

	hca->log_pg_sz = PAGE_SHIFT;

free:
	mcx_dmamem_free(sc, &mxm);

	return (error);
}


static int
mcx_init_hca(struct mcx_softc *sc)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_init_hca_in *in;
	struct mcx_cmd_init_hca_out *out;
	int error;
	uint8_t status;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_INIT_HCA);
	in->cmd_op_mod = htobe16(0);

	mcx_cmdq_post(sc, cqe, 0);

	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf(", hca init timeout\n");
		return (-1);
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf(", hca init command corrupt\n");
		return (-1);
	}

	status = cqe->cq_output_data[0];
	if (status != MCX_CQ_STATUS_OK) {
		printf(", hca init failed (%x)\n", status);
		return (-1);
	}

	return (0);
}

static int
mcx_set_driver_version(struct mcx_softc *sc)
{
	struct mcx_dmamem mxm;
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_set_driver_version_in *in;
	struct mcx_cmd_set_driver_version_out *out;
	int error;
	int token;
	uint8_t status;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in) +
	    sizeof(struct mcx_cmd_set_driver_version), sizeof(*out), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_SET_DRIVER_VERSION);
	in->cmd_op_mod = htobe16(0);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
	    &cqe->cq_input_ptr, token) != 0) {
		printf(", unable to allocate set driver version mailboxen\n");
		return (-1);
	}
	strlcpy(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)),
	    "OpenBSD,mcx,1.000.000000", MCX_CMDQ_MAILBOX_DATASIZE);

	mcx_cmdq_mboxes_sign(&mxm, 1);
	mcx_cmdq_post(sc, cqe, 0);

	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf(", set driver version timeout\n");
		goto free;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf(", set driver version command corrupt\n");
		goto free;
	}

	status = cqe->cq_output_data[0];
	if (status != MCX_CQ_STATUS_OK) {
		printf(", set driver version failed (%x)\n", status);
		error = -1;
		goto free;
	}

free:
	mcx_dmamem_free(sc, &mxm);

	return (error);
}

static int
mcx_iff(struct mcx_softc *sc)
{
	struct ifnet *ifp = &sc->sc_ac.ac_if;
	struct mcx_dmamem mxm;
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_modify_nic_vport_context_in *in;
	struct mcx_cmd_modify_nic_vport_context_out *out;
	struct mcx_nic_vport_ctx *ctx;
	int error;
	int token;
	int insize;
	uint32_t dest;

	dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE |
	    sc->sc_rss_flow_table_id;

	/* enable or disable the promisc flow */
	if (ISSET(ifp->if_flags, IFF_PROMISC)) {
		if (sc->sc_promisc_flow_enabled == 0) {
			mcx_set_flow_table_entry_mac(sc,
			    MCX_FLOW_GROUP_PROMISC, 0, NULL, dest);
			sc->sc_promisc_flow_enabled = 1;
		}
	} else if (sc->sc_promisc_flow_enabled != 0) {
		mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0);
		sc->sc_promisc_flow_enabled = 0;
	}

	/* enable or disable the all-multicast flow */
	if (ISSET(ifp->if_flags, IFF_ALLMULTI)) {
		if (sc->sc_allmulti_flow_enabled == 0) {
			uint8_t mcast[ETHER_ADDR_LEN];

			memset(mcast, 0, sizeof(mcast));
			mcast[0] = 0x01;
			mcx_set_flow_table_entry_mac(sc,
			    MCX_FLOW_GROUP_ALLMULTI, 0, mcast, dest);
			sc->sc_allmulti_flow_enabled = 1;
		}
	} else if (sc->sc_allmulti_flow_enabled != 0) {
		mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0);
		sc->sc_allmulti_flow_enabled = 0;
	}

	insize = sizeof(struct mcx_nic_vport_ctx) + 240;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_MODIFY_NIC_VPORT_CONTEXT);
	in->cmd_op_mod = htobe16(0);
	in->cmd_field_select = htobe32(
	    MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC |
	    MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) {
		printf(", unable to allocate modify "
		    "nic vport context mailboxen\n");
		return (-1);
	}
	ctx = (struct mcx_nic_vport_ctx *)
	    (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 240);
	ctx->vp_mtu = htobe32(sc->sc_hardmtu);
	/*
         * always leave promisc-all enabled on the vport since we
         * can't give it a vlan list, and we're already doing multicast
         * filtering in the flow table.
	 */
	ctx->vp_flags = htobe16(MCX_NIC_VPORT_CTX_PROMISC_ALL);

	mcx_cmdq_mboxes_sign(&mxm, 1);
	mcx_cmdq_post(sc, cqe, 0);

	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf(", modify nic vport context timeout\n");
		goto free;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf(", modify nic vport context command corrupt\n");
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf(", modify nic vport context failed (%x, %x)\n",
		    out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

free:
	mcx_dmamem_free(sc, &mxm);

	return (error);
}

static int
mcx_alloc_uar(struct mcx_softc *sc, int *uar)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_alloc_uar_in *in;
	struct mcx_cmd_alloc_uar_out *out;
	int error;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_ALLOC_UAR);
	in->cmd_op_mod = htobe16(0);

	mcx_cmdq_post(sc, cqe, 0);

	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf(", alloc uar timeout\n");
		return (-1);
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf(", alloc uar command corrupt\n");
		return (-1);
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf(", alloc uar failed (%x)\n", out->cmd_status);
		return (-1);
	}

	*uar = mcx_get_id(out->cmd_uar);
	return (0);
}

static int
mcx_create_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar,
    uint64_t events, int vector)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_dmamem mxm;
	struct mcx_cmd_create_eq_in *in;
	struct mcx_cmd_create_eq_mb_in *mbin;
	struct mcx_cmd_create_eq_out *out;
	struct mcx_eq_entry *eqe;
	int error;
	uint64_t *pas;
	int insize, npages, paslen, i, token;

	eq->eq_cons = 0;

	npages = howmany((1 << MCX_LOG_EQ_SIZE) * sizeof(struct mcx_eq_entry),
	    MCX_PAGE_SIZE);
	paslen = npages * sizeof(*pas);
	insize = sizeof(struct mcx_cmd_create_eq_mb_in) + paslen;

	if (mcx_dmamem_alloc(sc, &eq->eq_mem, npages * MCX_PAGE_SIZE,
	    MCX_PAGE_SIZE) != 0) {
		printf(", unable to allocate event queue memory\n");
		return (-1);
	}

	eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem);
	for (i = 0; i < (1 << MCX_LOG_EQ_SIZE); i++) {
		eqe[i].eq_owner = MCX_EQ_ENTRY_OWNER_INIT;
	}

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_CREATE_EQ);
	in->cmd_op_mod = htobe16(0);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm,
	    howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
	    &cqe->cq_input_ptr, token) != 0) {
		printf(", unable to allocate create eq mailboxen\n");
		goto free_eq;
	}
	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
	mbin->cmd_eq_ctx.eq_uar_size = htobe32(
	    (MCX_LOG_EQ_SIZE << MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT) | uar);
	mbin->cmd_eq_ctx.eq_intr = vector;
	mbin->cmd_event_bitmask = htobe64(events);

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
	    0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD);

	/* physical addresses follow the mailbox in data */
	mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &eq->eq_mem);
	mcx_cmdq_mboxes_sign(&mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE));
	mcx_cmdq_post(sc, cqe, 0);

	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf(", create eq timeout\n");
		goto free_mxm;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf(", create eq command corrupt\n");
		goto free_mxm;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf(", create eq failed (%x, %x)\n", out->cmd_status,
		    betoh32(out->cmd_syndrome));
		goto free_mxm;
	}

	eq->eq_n = mcx_get_id(out->cmd_eqn);

	mcx_dmamem_free(sc, &mxm);

	mcx_arm_eq(sc, eq, uar);

	return (0);

free_mxm:
	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
	    0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD);
	mcx_dmamem_free(sc, &mxm);
free_eq:
	mcx_dmamem_free(sc, &eq->eq_mem);
	return (-1);
}

static int
mcx_alloc_pd(struct mcx_softc *sc)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_alloc_pd_in *in;
	struct mcx_cmd_alloc_pd_out *out;
	int error;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_ALLOC_PD);
	in->cmd_op_mod = htobe16(0);

	mcx_cmdq_post(sc, cqe, 0);

	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf(", alloc pd timeout\n");
		return (-1);
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf(", alloc pd command corrupt\n");
		return (-1);
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf(", alloc pd failed (%x)\n", out->cmd_status);
		return (-1);
	}

	sc->sc_pd = mcx_get_id(out->cmd_pd);
	return (0);
}

static int
mcx_alloc_tdomain(struct mcx_softc *sc)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_alloc_td_in *in;
	struct mcx_cmd_alloc_td_out *out;
	int error;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_ALLOC_TRANSPORT_DOMAIN);
	in->cmd_op_mod = htobe16(0);

	mcx_cmdq_post(sc, cqe, 0);

	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf(", alloc transport domain timeout\n");
		return (-1);
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf(", alloc transport domain command corrupt\n");
		return (-1);
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf(", alloc transport domain failed (%x)\n",
		    out->cmd_status);
		return (-1);
	}

	sc->sc_tdomain = mcx_get_id(out->cmd_tdomain);
	return (0);
}

static int
mcx_query_nic_vport_context(struct mcx_softc *sc)
{
	struct mcx_dmamem mxm;
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_query_nic_vport_context_in *in;
	struct mcx_cmd_query_nic_vport_context_out *out;
	struct mcx_nic_vport_ctx *ctx;
	uint8_t *addr;
	int error, token, i;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_QUERY_NIC_VPORT_CONTEXT);
	in->cmd_op_mod = htobe16(0);
	in->cmd_allowed_list_type = 0;

	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
	    &cqe->cq_output_ptr, token) != 0) {
		printf(", unable to allocate "
		    "query nic vport context mailboxen\n");
		return (-1);
	}
	mcx_cmdq_mboxes_sign(&mxm, 1);
	mcx_cmdq_post(sc, cqe, 0);

	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf(", query nic vport context timeout\n");
		goto free;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf(", query nic vport context command corrupt\n");
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf(", query nic vport context failed (%x, %x)\n",
		    out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

	ctx = (struct mcx_nic_vport_ctx *)
	    mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
	addr = (uint8_t *)&ctx->vp_perm_addr;
	for (i = 0; i < ETHER_ADDR_LEN; i++) {
		sc->sc_ac.ac_enaddr[i] = addr[i + 2];
	}
free:
	mcx_dmamem_free(sc, &mxm);

	return (error);
}

static int
mcx_query_special_contexts(struct mcx_softc *sc)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_query_special_ctx_in *in;
	struct mcx_cmd_query_special_ctx_out *out;
	int error;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_QUERY_SPECIAL_CONTEXTS);
	in->cmd_op_mod = htobe16(0);

	mcx_cmdq_post(sc, cqe, 0);

	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf(", query special contexts timeout\n");
		return (-1);
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf(", query special contexts command corrupt\n");
		return (-1);
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf(", query special contexts failed (%x)\n",
		    out->cmd_status);
		return (-1);
	}

	sc->sc_lkey = betoh32(out->cmd_resd_lkey);
	return (0);
}

static int
mcx_set_port_mtu(struct mcx_softc *sc, int mtu)
{
	struct mcx_reg_pmtu pmtu;
	int error;

	/* read max mtu */
	memset(&pmtu, 0, sizeof(pmtu));
	pmtu.rp_local_port = 1;
	error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_READ, &pmtu,
	    sizeof(pmtu), MCX_CMDQ_SLOT_POLL);
	if (error != 0) {
		printf(", unable to get port MTU\n");
		return error;
	}

	mtu = min(mtu, betoh16(pmtu.rp_max_mtu));
	pmtu.rp_admin_mtu = htobe16(mtu);
	error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_WRITE, &pmtu,
	    sizeof(pmtu), MCX_CMDQ_SLOT_POLL);
	if (error != 0) {
		printf(", unable to set port MTU\n");
		return error;
	}

	sc->sc_hardmtu = mtu;
	sc->sc_rxbufsz = roundup(mtu + ETHER_ALIGN, sizeof(long));
	return 0;
}

static int
mcx_create_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar, int db, int eqn)
{
	struct mcx_cmdq_entry *cmde;
	struct mcx_cq_entry *cqe;
	struct mcx_dmamem mxm;
	struct mcx_cmd_create_cq_in *in;
	struct mcx_cmd_create_cq_mb_in *mbin;
	struct mcx_cmd_create_cq_out *out;
	int error;
	uint64_t *pas;
	int insize, npages, paslen, i, token;

	cq->cq_doorbell = MCX_CQ_DOORBELL_BASE + (MCX_CQ_DOORBELL_STRIDE * db);

	npages = howmany((1 << MCX_LOG_CQ_SIZE) * sizeof(struct mcx_cq_entry),
	    MCX_PAGE_SIZE);
	paslen = npages * sizeof(*pas);
	insize = sizeof(struct mcx_cmd_create_cq_mb_in) + paslen;

	if (mcx_dmamem_alloc(sc, &cq->cq_mem, npages * MCX_PAGE_SIZE,
	    MCX_PAGE_SIZE) != 0) {
		printf("%s: unable to allocate completion queue memory\n",
		    DEVNAME(sc));
		return (-1);
	}
	cqe = MCX_DMA_KVA(&cq->cq_mem);
	for (i = 0; i < (1 << MCX_LOG_CQ_SIZE); i++) {
		cqe[i].cq_opcode_owner = MCX_CQ_ENTRY_FLAG_OWNER;
	}

	cmde = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cmde, sizeof(*in) + insize, sizeof(*out), token);

	in = mcx_cmdq_in(cmde);
	in->cmd_opcode = htobe16(MCX_CMD_CREATE_CQ);
	in->cmd_op_mod = htobe16(0);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm,
	    howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
	    &cmde->cq_input_ptr, token) != 0) {
		printf("%s: unable to allocate create cq mailboxen\n",
		    DEVNAME(sc));
		goto free_cq;
	}
	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
	mbin->cmd_cq_ctx.cq_uar_size = htobe32(
	    (MCX_LOG_CQ_SIZE << MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT) | uar);
	mbin->cmd_cq_ctx.cq_eqn = htobe32(eqn);
	mbin->cmd_cq_ctx.cq_period_max_count = htobe32(
	    (MCX_CQ_MOD_PERIOD << MCX_CQ_CTX_PERIOD_SHIFT) |
	    MCX_CQ_MOD_COUNTER);
	mbin->cmd_cq_ctx.cq_doorbell = htobe64(
	    MCX_DMA_DVA(&sc->sc_doorbell_mem) + cq->cq_doorbell);

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
	    0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_PREREAD);

	/* physical addresses follow the mailbox in data */
	mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &cq->cq_mem);
	mcx_cmdq_post(sc, cmde, 0);

	error = mcx_cmdq_poll(sc, cmde, 1000);
	if (error != 0) {
		printf("%s: create cq timeout\n", DEVNAME(sc));
		goto free_mxm;
	}
	if (mcx_cmdq_verify(cmde) != 0) {
		printf("%s: create cq command corrupt\n", DEVNAME(sc));
		goto free_mxm;
	}

	out = mcx_cmdq_out(cmde);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: create cq failed (%x, %x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		goto free_mxm;
	}

	cq->cq_n = mcx_get_id(out->cmd_cqn);
	cq->cq_cons = 0;
	cq->cq_count = 0;

	mcx_dmamem_free(sc, &mxm);

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
	    cq->cq_doorbell, sizeof(struct mcx_cq_doorbell),
	    BUS_DMASYNC_PREWRITE);

	mcx_arm_cq(sc, cq, uar);

	return (0);

free_mxm:
	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
	    0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD);
	mcx_dmamem_free(sc, &mxm);
free_cq:
	mcx_dmamem_free(sc, &cq->cq_mem);
	return (-1);
}

static int
mcx_destroy_cq(struct mcx_softc *sc, struct mcx_cq *cq)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_destroy_cq_in *in;
	struct mcx_cmd_destroy_cq_out *out;
	int error;
	int token;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_CQ);
	in->cmd_op_mod = htobe16(0);
	in->cmd_cqn = htobe32(cq->cq_n);

	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: destroy cq timeout\n", DEVNAME(sc));
		return error;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: destroy cq command corrupt\n", DEVNAME(sc));
		return error;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: destroy cq failed (%x, %x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		return -1;
	}

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
	    cq->cq_doorbell, sizeof(struct mcx_cq_doorbell),
	    BUS_DMASYNC_POSTWRITE);

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
	    0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD);
	mcx_dmamem_free(sc, &cq->cq_mem);

	cq->cq_n = 0;
	cq->cq_cons = 0;
	cq->cq_count = 0;
	return 0;
}

static int
mcx_create_rq(struct mcx_softc *sc, struct mcx_rx *rx, int db, int cqn)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_dmamem mxm;
	struct mcx_cmd_create_rq_in *in;
	struct mcx_cmd_create_rq_out *out;
	struct mcx_rq_ctx *mbin;
	int error;
	uint64_t *pas;
	uint32_t rq_flags;
	int insize, npages, paslen, token;

	rx->rx_doorbell = MCX_WQ_DOORBELL_BASE +
	    (db * MCX_WQ_DOORBELL_STRIDE);

	npages = howmany((1 << MCX_LOG_RQ_SIZE) * sizeof(struct mcx_rq_entry),
	    MCX_PAGE_SIZE);
	paslen = npages * sizeof(*pas);
	insize = 0x10 + sizeof(struct mcx_rq_ctx) + paslen;

	if (mcx_dmamem_alloc(sc, &rx->rx_rq_mem, npages * MCX_PAGE_SIZE,
	    MCX_PAGE_SIZE) != 0) {
		printf("%s: unable to allocate receive queue memory\n",
		    DEVNAME(sc));
		return (-1);
	}

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQ);
	in->cmd_op_mod = htobe16(0);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm,
	    howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
	    &cqe->cq_input_ptr, token) != 0) {
		printf("%s: unable to allocate create rq mailboxen\n",
		    DEVNAME(sc));
		goto free_rq;
	}
	mbin = (struct mcx_rq_ctx *)
	    (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10);
	rq_flags = MCX_RQ_CTX_RLKEY;
#if NVLAN == 0
	rq_flags |= MCX_RQ_CTX_VLAN_STRIP_DIS;
#endif
	mbin->rq_flags = htobe32(rq_flags);
	mbin->rq_cqn = htobe32(cqn);
	mbin->rq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC;
	mbin->rq_wq.wq_pd = htobe32(sc->sc_pd);
	mbin->rq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) +
	    rx->rx_doorbell);
	mbin->rq_wq.wq_log_stride = htobe16(4);
	mbin->rq_wq.wq_log_size = MCX_LOG_RQ_SIZE;

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
	    0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_PREWRITE);

	/* physical addresses follow the mailbox in data */
	mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10, npages, &rx->rx_rq_mem);
	mcx_cmdq_post(sc, cqe, 0);

	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: create rq timeout\n", DEVNAME(sc));
		goto free_mxm;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: create rq command corrupt\n", DEVNAME(sc));
		goto free_mxm;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: create rq failed (%x, %x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		goto free_mxm;
	}

	rx->rx_rqn = mcx_get_id(out->cmd_rqn);

	mcx_dmamem_free(sc, &mxm);

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
	    rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);

	return (0);

free_mxm:
	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
	    0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE);
	mcx_dmamem_free(sc, &mxm);
free_rq:
	mcx_dmamem_free(sc, &rx->rx_rq_mem);
	return (-1);
}

static int
mcx_ready_rq(struct mcx_softc *sc, struct mcx_rx *rx)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_dmamem mxm;
	struct mcx_cmd_modify_rq_in *in;
	struct mcx_cmd_modify_rq_mb_in *mbin;
	struct mcx_cmd_modify_rq_out *out;
	int error;
	int token;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
	    sizeof(*out), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_MODIFY_RQ);
	in->cmd_op_mod = htobe16(0);
	in->cmd_rq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | rx->rx_rqn);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
	    &cqe->cq_input_ptr, token) != 0) {
		printf("%s: unable to allocate modify rq mailbox\n",
		    DEVNAME(sc));
		return (-1);
	}
	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
	mbin->cmd_rq_ctx.rq_flags = htobe32(
	    MCX_QUEUE_STATE_RDY << MCX_RQ_CTX_STATE_SHIFT);

	mcx_cmdq_mboxes_sign(&mxm, 1);
	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: modify rq timeout\n", DEVNAME(sc));
		goto free;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: modify rq command corrupt\n", DEVNAME(sc));
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: modify rq failed (%x, %x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

free:
	mcx_dmamem_free(sc, &mxm);
	return (error);
}

static int
mcx_destroy_rq(struct mcx_softc *sc, struct mcx_rx *rx)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_destroy_rq_in *in;
	struct mcx_cmd_destroy_rq_out *out;
	int error;
	int token;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQ);
	in->cmd_op_mod = htobe16(0);
	in->cmd_rqn = htobe32(rx->rx_rqn);

	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: destroy rq timeout\n", DEVNAME(sc));
		return error;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: destroy rq command corrupt\n", DEVNAME(sc));
		return error;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: destroy rq failed (%x, %x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		return -1;
	}

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
	    rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
	    0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE);
	mcx_dmamem_free(sc, &rx->rx_rq_mem);

	rx->rx_rqn = 0;
	return 0;
}

static int
mcx_create_tir_direct(struct mcx_softc *sc, struct mcx_rx *rx, int *tirn)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_dmamem mxm;
	struct mcx_cmd_create_tir_in *in;
	struct mcx_cmd_create_tir_mb_in *mbin;
	struct mcx_cmd_create_tir_out *out;
	int error;
	int token;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
	    sizeof(*out), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR);
	in->cmd_op_mod = htobe16(0);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
	    &cqe->cq_input_ptr, token) != 0) {
		printf("%s: unable to allocate create tir mailbox\n",
		    DEVNAME(sc));
		return (-1);
	}
	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
	/* leave disp_type = 0, so packets get sent to the inline rqn */
	mbin->cmd_inline_rqn = htobe32(rx->rx_rqn);
	mbin->cmd_tdomain = htobe32(sc->sc_tdomain);

	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: create tir timeout\n", DEVNAME(sc));
		goto free;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: create tir command corrupt\n", DEVNAME(sc));
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

	*tirn = mcx_get_id(out->cmd_tirn);
free:
	mcx_dmamem_free(sc, &mxm);
	return (error);
}

static int
mcx_create_tir_indirect(struct mcx_softc *sc, int rqtn, uint32_t hash_sel,
    int *tirn)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_dmamem mxm;
	struct mcx_cmd_create_tir_in *in;
	struct mcx_cmd_create_tir_mb_in *mbin;
	struct mcx_cmd_create_tir_out *out;
	int error;
	int token;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
	    sizeof(*out), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR);
	in->cmd_op_mod = htobe16(0);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
	    &cqe->cq_input_ptr, token) != 0) {
		printf("%s: unable to allocate create tir mailbox\n",
		    DEVNAME(sc));
		return (-1);
	}
	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
	mbin->cmd_disp_type = htobe32(MCX_TIR_CTX_DISP_TYPE_INDIRECT
	    << MCX_TIR_CTX_DISP_TYPE_SHIFT);
	mbin->cmd_indir_table = htobe32(rqtn);
	mbin->cmd_tdomain = htobe32(sc->sc_tdomain |
	    MCX_TIR_CTX_HASH_TOEPLITZ << MCX_TIR_CTX_HASH_SHIFT);
	mbin->cmd_rx_hash_sel_outer = htobe32(hash_sel);
	stoeplitz_to_key(&mbin->cmd_rx_hash_key,
	    sizeof(mbin->cmd_rx_hash_key));

	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: create tir timeout\n", DEVNAME(sc));
		goto free;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: create tir command corrupt\n", DEVNAME(sc));
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

	*tirn = mcx_get_id(out->cmd_tirn);
free:
	mcx_dmamem_free(sc, &mxm);
	return (error);
}

static int
mcx_destroy_tir(struct mcx_softc *sc, int tirn)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_destroy_tir_in *in;
	struct mcx_cmd_destroy_tir_out *out;
	int error;
	int token;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIR);
	in->cmd_op_mod = htobe16(0);
	in->cmd_tirn = htobe32(tirn);

	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: destroy tir timeout\n", DEVNAME(sc));
		return error;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: destroy tir command corrupt\n", DEVNAME(sc));
		return error;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: destroy tir failed (%x, %x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		return -1;
	}

	return (0);
}

static int
mcx_create_sq(struct mcx_softc *sc, struct mcx_tx *tx, int uar, int db,
    int cqn)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_dmamem mxm;
	struct mcx_cmd_create_sq_in *in;
	struct mcx_sq_ctx *mbin;
	struct mcx_cmd_create_sq_out *out;
	int error;
	uint64_t *pas;
	int insize, npages, paslen, token;

	tx->tx_doorbell = MCX_WQ_DOORBELL_BASE +
	    (db * MCX_WQ_DOORBELL_STRIDE) + 4;

	npages = howmany((1 << MCX_LOG_SQ_SIZE) * sizeof(struct mcx_sq_entry),
	    MCX_PAGE_SIZE);
	paslen = npages * sizeof(*pas);
	insize = sizeof(struct mcx_sq_ctx) + paslen;

	if (mcx_dmamem_alloc(sc, &tx->tx_sq_mem, npages * MCX_PAGE_SIZE,
	    MCX_PAGE_SIZE) != 0) {
		printf("%s: unable to allocate send queue memory\n",
		    DEVNAME(sc));
		return (-1);
	}

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in) + insize + paslen, sizeof(*out),
	    token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_CREATE_SQ);
	in->cmd_op_mod = htobe16(0);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm,
	    howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
	    &cqe->cq_input_ptr, token) != 0) {
		printf("%s: unable to allocate create sq mailboxen\n",
		    DEVNAME(sc));
		goto free_sq;
	}
	mbin = (struct mcx_sq_ctx *)
	    (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10);
	mbin->sq_flags = htobe32(MCX_SQ_CTX_RLKEY |
	    (1 << MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT));
	mbin->sq_cqn = htobe32(cqn);
	mbin->sq_tis_lst_sz = htobe32(1 << MCX_SQ_CTX_TIS_LST_SZ_SHIFT);
	mbin->sq_tis_num = htobe32(sc->sc_tis);
	mbin->sq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC;
	mbin->sq_wq.wq_pd = htobe32(sc->sc_pd);
	mbin->sq_wq.wq_uar_page = htobe32(uar);
	mbin->sq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) +
	    tx->tx_doorbell);
	mbin->sq_wq.wq_log_stride = htobe16(MCX_LOG_SQ_ENTRY_SIZE);
	mbin->sq_wq.wq_log_size = MCX_LOG_SQ_SIZE;

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
	    0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_PREWRITE);

	/* physical addresses follow the mailbox in data */
	mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10,
	    npages, &tx->tx_sq_mem);
	mcx_cmdq_post(sc, cqe, 0);

	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: create sq timeout\n", DEVNAME(sc));
		goto free_mxm;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: create sq command corrupt\n", DEVNAME(sc));
		goto free_mxm;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: create sq failed (%x, %x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		goto free_mxm;
	}

	tx->tx_uar = uar;
	tx->tx_sqn = mcx_get_id(out->cmd_sqn);

	mcx_dmamem_free(sc, &mxm);

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
	    tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);

	return (0);

free_mxm:
	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
	    0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE);
	mcx_dmamem_free(sc, &mxm);
free_sq:
	mcx_dmamem_free(sc, &tx->tx_sq_mem);
	return (-1);
}

static int
mcx_destroy_sq(struct mcx_softc *sc, struct mcx_tx *tx)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_destroy_sq_in *in;
	struct mcx_cmd_destroy_sq_out *out;
	int error;
	int token;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_SQ);
	in->cmd_op_mod = htobe16(0);
	in->cmd_sqn = htobe32(tx->tx_sqn);

	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: destroy sq timeout\n", DEVNAME(sc));
		return error;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: destroy sq command corrupt\n", DEVNAME(sc));
		return error;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: destroy sq failed (%x, %x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		return -1;
	}

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
	    tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
	    0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE);
	mcx_dmamem_free(sc, &tx->tx_sq_mem);

	tx->tx_sqn = 0;
	return 0;
}

static int
mcx_ready_sq(struct mcx_softc *sc, struct mcx_tx *tx)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_dmamem mxm;
	struct mcx_cmd_modify_sq_in *in;
	struct mcx_cmd_modify_sq_mb_in *mbin;
	struct mcx_cmd_modify_sq_out *out;
	int error;
	int token;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
	    sizeof(*out), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_MODIFY_SQ);
	in->cmd_op_mod = htobe16(0);
	in->cmd_sq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | tx->tx_sqn);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
	    &cqe->cq_input_ptr, token) != 0) {
		printf("%s: unable to allocate modify sq mailbox\n",
		    DEVNAME(sc));
		return (-1);
	}
	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
	mbin->cmd_sq_ctx.sq_flags = htobe32(
	    MCX_QUEUE_STATE_RDY << MCX_SQ_CTX_STATE_SHIFT);

	mcx_cmdq_mboxes_sign(&mxm, 1);
	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: modify sq timeout\n", DEVNAME(sc));
		goto free;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: modify sq command corrupt\n", DEVNAME(sc));
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: modify sq failed (%x, %x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

free:
	mcx_dmamem_free(sc, &mxm);
	return (error);
}

static int
mcx_create_tis(struct mcx_softc *sc, int *tis)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_dmamem mxm;
	struct mcx_cmd_create_tis_in *in;
	struct mcx_cmd_create_tis_mb_in *mbin;
	struct mcx_cmd_create_tis_out *out;
	int error;
	int token;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
	    sizeof(*out), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIS);
	in->cmd_op_mod = htobe16(0);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
	    &cqe->cq_input_ptr, token) != 0) {
		printf("%s: unable to allocate create tis mailbox\n",
		    DEVNAME(sc));
		return (-1);
	}
	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
	mbin->cmd_tdomain = htobe32(sc->sc_tdomain);

	mcx_cmdq_mboxes_sign(&mxm, 1);
	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: create tis timeout\n", DEVNAME(sc));
		goto free;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: create tis command corrupt\n", DEVNAME(sc));
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: create tis failed (%x, %x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

	*tis = mcx_get_id(out->cmd_tisn);
free:
	mcx_dmamem_free(sc, &mxm);
	return (error);
}

static int
mcx_destroy_tis(struct mcx_softc *sc, int tis)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_destroy_tis_in *in;
	struct mcx_cmd_destroy_tis_out *out;
	int error;
	int token;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIS);
	in->cmd_op_mod = htobe16(0);
	in->cmd_tisn = htobe32(tis);

	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: destroy tis timeout\n", DEVNAME(sc));
		return error;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: destroy tis command corrupt\n", DEVNAME(sc));
		return error;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: destroy tis failed (%x, %x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		return -1;
	}

	return 0;
}

static int
mcx_create_rqt(struct mcx_softc *sc, int size, int *rqns, int *rqt)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_dmamem mxm;
	struct mcx_cmd_create_rqt_in *in;
	struct mcx_cmd_create_rqt_mb_in *mbin;
	struct mcx_cmd_create_rqt_out *out;
	struct mcx_rqt_ctx *rqt_ctx;
	int *rqtn;
	int error;
	int token;
	int i;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) +
	    (size * sizeof(int)), sizeof(*out), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQT);
	in->cmd_op_mod = htobe16(0);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
	    &cqe->cq_input_ptr, token) != 0) {
		printf("%s: unable to allocate create rqt mailbox\n",
		    DEVNAME(sc));
		return (-1);
	}
	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
	rqt_ctx = &mbin->cmd_rqt;
	rqt_ctx->cmd_rqt_max_size = htobe16(sc->sc_max_rqt_size);
	rqt_ctx->cmd_rqt_actual_size = htobe16(size);

	/* rqt list follows the rqt context */
	rqtn = (int *)(rqt_ctx + 1);
	for (i = 0; i < size; i++) {
		rqtn[i] = htobe32(rqns[i]);
	}

	mcx_cmdq_mboxes_sign(&mxm, 1);
	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: create rqt timeout\n", DEVNAME(sc));
		goto free;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: create rqt command corrupt\n", DEVNAME(sc));
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: create rqt failed (%x, %x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

	*rqt = mcx_get_id(out->cmd_rqtn);
	return (0);
free:
	mcx_dmamem_free(sc, &mxm);
	return (error);
}

static int
mcx_destroy_rqt(struct mcx_softc *sc, int rqt)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_destroy_rqt_in *in;
	struct mcx_cmd_destroy_rqt_out *out;
	int error;
	int token;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQT);
	in->cmd_op_mod = htobe16(0);
	in->cmd_rqtn = htobe32(rqt);

	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: destroy rqt timeout\n", DEVNAME(sc));
		return error;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: destroy rqt command corrupt\n", DEVNAME(sc));
		return error;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: destroy rqt failed (%x, %x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		return -1;
	}

	return 0;
}

#if 0
static int
mcx_alloc_flow_counter(struct mcx_softc *sc, int i)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_alloc_flow_counter_in *in;
	struct mcx_cmd_alloc_flow_counter_out *out;
	int error;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_ALLOC_FLOW_COUNTER);
	in->cmd_op_mod = htobe16(0);

	mcx_cmdq_post(sc, cqe, 0);

	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: alloc flow counter timeout\n", DEVNAME(sc));
		return (-1);
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: alloc flow counter command corrupt\n", DEVNAME(sc));
		return (-1);
	}

	out = (struct mcx_cmd_alloc_flow_counter_out *)cqe->cq_output_data;
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: alloc flow counter failed (%x)\n", DEVNAME(sc),
		    out->cmd_status);
		return (-1);
	}

	sc->sc_flow_counter_id[i]  = betoh16(out->cmd_flow_counter_id);
	printf("flow counter id %d = %d\n", i, sc->sc_flow_counter_id[i]);

	return (0);
}
#endif

static int
mcx_create_flow_table(struct mcx_softc *sc, int log_size, int level,
    int *flow_table_id)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_dmamem mxm;
	struct mcx_cmd_create_flow_table_in *in;
	struct mcx_cmd_create_flow_table_mb_in *mbin;
	struct mcx_cmd_create_flow_table_out *out;
	int error;
	int token;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
	    sizeof(*out), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_TABLE);
	in->cmd_op_mod = htobe16(0);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
	    &cqe->cq_input_ptr, token) != 0) {
		printf("%s: unable to allocate create flow table mailbox\n",
		    DEVNAME(sc));
		return (-1);
	}
	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
	mbin->cmd_ctx.ft_log_size = log_size;
	mbin->cmd_ctx.ft_level = level;

	mcx_cmdq_mboxes_sign(&mxm, 1);
	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: create flow table timeout\n", DEVNAME(sc));
		goto free;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: create flow table command corrupt\n", DEVNAME(sc));
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: create flow table failed (%x, %x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

	*flow_table_id = mcx_get_id(out->cmd_table_id);
free:
	mcx_dmamem_free(sc, &mxm);
	return (error);
}

static int
mcx_set_flow_table_root(struct mcx_softc *sc, int flow_table_id)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_dmamem mxm;
	struct mcx_cmd_set_flow_table_root_in *in;
	struct mcx_cmd_set_flow_table_root_mb_in *mbin;
	struct mcx_cmd_set_flow_table_root_out *out;
	int error;
	int token;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
	    sizeof(*out), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ROOT);
	in->cmd_op_mod = htobe16(0);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
	    &cqe->cq_input_ptr, token) != 0) {
		printf("%s: unable to allocate set flow table root mailbox\n",
		    DEVNAME(sc));
		return (-1);
	}
	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
	mbin->cmd_table_id = htobe32(flow_table_id);

	mcx_cmdq_mboxes_sign(&mxm, 1);
	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: set flow table root timeout\n", DEVNAME(sc));
		goto free;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: set flow table root command corrupt\n",
		    DEVNAME(sc));
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: set flow table root failed (%x, %x)\n",
		    DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

free:
	mcx_dmamem_free(sc, &mxm);
	return (error);
}

static int
mcx_destroy_flow_table(struct mcx_softc *sc, int flow_table_id)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_dmamem mxm;
	struct mcx_cmd_destroy_flow_table_in *in;
	struct mcx_cmd_destroy_flow_table_mb_in *mb;
	struct mcx_cmd_destroy_flow_table_out *out;
	int error;
	int token;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_TABLE);
	in->cmd_op_mod = htobe16(0);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
	    &cqe->cq_input_ptr, token) != 0) {
		printf("%s: unable to allocate destroy flow table mailbox\n",
		    DEVNAME(sc));
		return (-1);
	}
	mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
	mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
	mb->cmd_table_id = htobe32(flow_table_id);

	mcx_cmdq_mboxes_sign(&mxm, 1);
	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: destroy flow table timeout\n", DEVNAME(sc));
		goto free;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: destroy flow table command corrupt\n",
		    DEVNAME(sc));
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: destroy flow table failed (%x, %x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

free:
	mcx_dmamem_free(sc, &mxm);
	return (error);
}


static int
mcx_create_flow_group(struct mcx_softc *sc, int flow_table_id, int group,
    int start, int size, int match_enable, struct mcx_flow_match *match)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_dmamem mxm;
	struct mcx_cmd_create_flow_group_in *in;
	struct mcx_cmd_create_flow_group_mb_in *mbin;
	struct mcx_cmd_create_flow_group_out *out;
	struct mcx_flow_group *mfg;
	int error;
	int token;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out),
	    token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_GROUP);
	in->cmd_op_mod = htobe16(0);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token)
	    != 0) {
		printf("%s: unable to allocate create flow group mailbox\n",
		    DEVNAME(sc));
		return (-1);
	}
	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
	mbin->cmd_table_id = htobe32(flow_table_id);
	mbin->cmd_start_flow_index = htobe32(start);
	mbin->cmd_end_flow_index = htobe32(start + (size - 1));

	mbin->cmd_match_criteria_enable = match_enable;
	memcpy(&mbin->cmd_match_criteria, match, sizeof(*match));

	mcx_cmdq_mboxes_sign(&mxm, 2);
	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: create flow group timeout\n", DEVNAME(sc));
		goto free;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: create flow group command corrupt\n", DEVNAME(sc));
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: create flow group failed (%x, %x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

	mfg = &sc->sc_flow_group[group];
	mfg->g_id = mcx_get_id(out->cmd_group_id);
	mfg->g_table = flow_table_id;
	mfg->g_start = start;
	mfg->g_size = size;

free:
	mcx_dmamem_free(sc, &mxm);
	return (error);
}

static int
mcx_destroy_flow_group(struct mcx_softc *sc, int group)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_dmamem mxm;
	struct mcx_cmd_destroy_flow_group_in *in;
	struct mcx_cmd_destroy_flow_group_mb_in *mb;
	struct mcx_cmd_destroy_flow_group_out *out;
	struct mcx_flow_group *mfg;
	int error;
	int token;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_GROUP);
	in->cmd_op_mod = htobe16(0);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
	    &cqe->cq_input_ptr, token) != 0) {
		printf("%s: unable to allocate destroy flow group mailbox\n",
		    DEVNAME(sc));
		return (-1);
	}
	mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
	mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
	mfg = &sc->sc_flow_group[group];
	mb->cmd_table_id = htobe32(mfg->g_table);
	mb->cmd_group_id = htobe32(mfg->g_id);

	mcx_cmdq_mboxes_sign(&mxm, 2);
	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: destroy flow group timeout\n", DEVNAME(sc));
		goto free;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: destroy flow group command corrupt\n", DEVNAME(sc));
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: destroy flow group failed (%x, %x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

	mfg->g_id = -1;
	mfg->g_table = -1;
	mfg->g_size = 0;
	mfg->g_start = 0;
free:
	mcx_dmamem_free(sc, &mxm);
	return (error);
}

static int
mcx_set_flow_table_entry_mac(struct mcx_softc *sc, int group, int index,
    uint8_t *macaddr, uint32_t dest)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_dmamem mxm;
	struct mcx_cmd_set_flow_table_entry_in *in;
	struct mcx_cmd_set_flow_table_entry_mb_in *mbin;
	struct mcx_cmd_set_flow_table_entry_out *out;
	struct mcx_flow_group *mfg;
	uint32_t *pdest;
	int error;
	int token;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest),
	    sizeof(*out), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY);
	in->cmd_op_mod = htobe16(0);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token)
	    != 0) {
		printf("%s: unable to allocate set flow table entry mailbox\n",
		    DEVNAME(sc));
		return (-1);
	}

	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;

	mfg = &sc->sc_flow_group[group];
	mbin->cmd_table_id = htobe32(mfg->g_table);
	mbin->cmd_flow_index = htobe32(mfg->g_start + index);
	mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id);

	/* flow context ends at offset 0x330, 0x130 into the second mbox */
	pdest = (uint32_t *)
	    (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130);
	mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD);
	mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1);
	*pdest = htobe32(dest);

	/* the only thing we match on at the moment is the dest mac address */
	if (macaddr != NULL) {
		memcpy(mbin->cmd_flow_ctx.fc_match_value.mc_dest_mac, macaddr,
		    ETHER_ADDR_LEN);
	}

	mcx_cmdq_mboxes_sign(&mxm, 2);
	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: set flow table entry timeout\n", DEVNAME(sc));
		goto free;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: set flow table entry command corrupt\n",
		    DEVNAME(sc));
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: set flow table entry failed (%x, %x)\n",
		    DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

free:
	mcx_dmamem_free(sc, &mxm);
	return (error);
}

static int
mcx_set_flow_table_entry_proto(struct mcx_softc *sc, int group, int index,
    int ethertype, int ip_proto, uint32_t dest)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_dmamem mxm;
	struct mcx_cmd_set_flow_table_entry_in *in;
	struct mcx_cmd_set_flow_table_entry_mb_in *mbin;
	struct mcx_cmd_set_flow_table_entry_out *out;
	struct mcx_flow_group *mfg;
	uint32_t *pdest;
	int error;
	int token;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest),
	    sizeof(*out), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY);
	in->cmd_op_mod = htobe16(0);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token)
	    != 0) {
		printf("%s: unable to allocate set flow table entry mailbox\n",
		    DEVNAME(sc));
		return (-1);
	}

	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;

	mfg = &sc->sc_flow_group[group];
	mbin->cmd_table_id = htobe32(mfg->g_table);
	mbin->cmd_flow_index = htobe32(mfg->g_start + index);
	mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id);

	/* flow context ends at offset 0x330, 0x130 into the second mbox */
	pdest = (uint32_t *)
	    (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130);
	mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD);
	mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1);
	*pdest = htobe32(dest);

	mbin->cmd_flow_ctx.fc_match_value.mc_ethertype = htobe16(ethertype);
	mbin->cmd_flow_ctx.fc_match_value.mc_ip_proto = ip_proto;

	mcx_cmdq_mboxes_sign(&mxm, 2);
	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: set flow table entry timeout\n", DEVNAME(sc));
		goto free;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: set flow table entry command corrupt\n",
		    DEVNAME(sc));
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: set flow table entry failed (%x, %x)\n",
		    DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

free:
	mcx_dmamem_free(sc, &mxm);
	return (error);
}

static int
mcx_delete_flow_table_entry(struct mcx_softc *sc, int group, int index)
{
	struct mcx_cmdq_entry *cqe;
	struct mcx_dmamem mxm;
	struct mcx_cmd_delete_flow_table_entry_in *in;
	struct mcx_cmd_delete_flow_table_entry_mb_in *mbin;
	struct mcx_cmd_delete_flow_table_entry_out *out;
	struct mcx_flow_group *mfg;
	int error;
	int token;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out),
	    token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_DELETE_FLOW_TABLE_ENTRY);
	in->cmd_op_mod = htobe16(0);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
	    &cqe->cq_input_ptr, token) != 0) {
		printf("%s: unable to allocate "
		    "delete flow table entry mailbox\n", DEVNAME(sc));
		return (-1);
	}
	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;

	mfg = &sc->sc_flow_group[group];
	mbin->cmd_table_id = htobe32(mfg->g_table);
	mbin->cmd_flow_index = htobe32(mfg->g_start + index);

	mcx_cmdq_mboxes_sign(&mxm, 2);
	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: delete flow table entry timeout\n", DEVNAME(sc));
		goto free;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: delete flow table entry command corrupt\n",
		    DEVNAME(sc));
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: delete flow table entry %d:%d failed (%x, %x)\n",
		    DEVNAME(sc), group, index, out->cmd_status,
		    betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

free:
	mcx_dmamem_free(sc, &mxm);
	return (error);
}

#if 0
int
mcx_dump_flow_table(struct mcx_softc *sc, int flow_table_id)
{
	struct mcx_dmamem mxm;
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_query_flow_table_in *in;
	struct mcx_cmd_query_flow_table_mb_in *mbin;
	struct mcx_cmd_query_flow_table_out *out;
	struct mcx_cmd_query_flow_table_mb_out *mbout;
	uint8_t token = mcx_cmdq_token(sc);
	int error;
	int i;
	uint8_t *dump;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
	    sizeof(*out) + sizeof(*mbout) + 16, token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE);
	in->cmd_op_mod = htobe16(0);

	CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE);
	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
	    &cqe->cq_output_ptr, token) != 0) {
		printf(", unable to allocate query flow table mailboxes\n");
		return (-1);
	}
	cqe->cq_input_ptr = cqe->cq_output_ptr;

	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
	mbin->cmd_table_type = 0;
	mbin->cmd_table_id = htobe32(flow_table_id);

	mcx_cmdq_mboxes_sign(&mxm, 1);

	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: query flow table timeout\n", DEVNAME(sc));
		goto free;
	}
	error = mcx_cmdq_verify(cqe);
	if (error != 0) {
		printf("%s: query flow table reply corrupt\n", DEVNAME(sc));
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	switch (out->cmd_status) {
	case MCX_CQ_STATUS_OK:
		break;
	default:
		printf("%s: query flow table failed (%x/%x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

        mbout = (struct mcx_cmd_query_flow_table_mb_out *)
	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
	dump = (uint8_t *)mbout + 8;
	for (i = 0; i < sizeof(struct mcx_flow_table_ctx); i++) {
		printf("%.2x ", dump[i]);
		if (i % 16 == 15)
			printf("\n");
	}
free:
	mcx_cq_mboxes_free(sc, &mxm);
	return (error);
}
int
mcx_dump_flow_table_entry(struct mcx_softc *sc, int flow_table_id, int index)
{
	struct mcx_dmamem mxm;
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_query_flow_table_entry_in *in;
	struct mcx_cmd_query_flow_table_entry_mb_in *mbin;
	struct mcx_cmd_query_flow_table_entry_out *out;
	struct mcx_cmd_query_flow_table_entry_mb_out *mbout;
	uint8_t token = mcx_cmdq_token(sc);
	int error;
	int i;
	uint8_t *dump;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
	    sizeof(*out) + sizeof(*mbout) + 16, token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE_ENTRY);
	in->cmd_op_mod = htobe16(0);

	CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
	    &cqe->cq_output_ptr, token) != 0) {
		printf(", unable to allocate "
		    "query flow table entry mailboxes\n");
		return (-1);
	}
	cqe->cq_input_ptr = cqe->cq_output_ptr;

	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
	mbin->cmd_table_type = 0;
	mbin->cmd_table_id = htobe32(flow_table_id);
	mbin->cmd_flow_index = htobe32(index);

	mcx_cmdq_mboxes_sign(&mxm, 1);

	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: query flow table entry timeout\n", DEVNAME(sc));
		goto free;
	}
	error = mcx_cmdq_verify(cqe);
	if (error != 0) {
		printf("%s: query flow table entry reply corrupt\n",
		    DEVNAME(sc));
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	switch (out->cmd_status) {
	case MCX_CQ_STATUS_OK:
		break;
	default:
		printf("%s: query flow table entry failed (%x/%x)\n",
		    DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

        mbout = (struct mcx_cmd_query_flow_table_entry_mb_out *)
	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
	dump = (uint8_t *)mbout;
	for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
		printf("%.2x ", dump[i]);
		if (i % 16 == 15)
			printf("\n");
	}

free:
	mcx_cq_mboxes_free(sc, &mxm);
	return (error);
}

int
mcx_dump_flow_group(struct mcx_softc *sc, int flow_table_id)
{
	struct mcx_dmamem mxm;
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_query_flow_group_in *in;
	struct mcx_cmd_query_flow_group_mb_in *mbin;
	struct mcx_cmd_query_flow_group_out *out;
	struct mcx_cmd_query_flow_group_mb_out *mbout;
	uint8_t token = mcx_cmdq_token(sc);
	int error;
	int i;
	uint8_t *dump;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
	    sizeof(*out) + sizeof(*mbout) + 16, token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_GROUP);
	in->cmd_op_mod = htobe16(0);

	CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
	    &cqe->cq_output_ptr, token) != 0) {
		printf(", unable to allocate query flow group mailboxes\n");
		return (-1);
	}
	cqe->cq_input_ptr = cqe->cq_output_ptr;

	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
	mbin->cmd_table_type = 0;
	mbin->cmd_table_id = htobe32(flow_table_id);
	mbin->cmd_group_id = htobe32(sc->sc_flow_group_id);

	mcx_cmdq_mboxes_sign(&mxm, 1);

	mcx_cmdq_post(sc, cqe, 0);
	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: query flow group timeout\n", DEVNAME(sc));
		goto free;
	}
	error = mcx_cmdq_verify(cqe);
	if (error != 0) {
		printf("%s: query flow group reply corrupt\n", DEVNAME(sc));
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	switch (out->cmd_status) {
	case MCX_CQ_STATUS_OK:
		break;
	default:
		printf("%s: query flow group failed (%x/%x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

        mbout = (struct mcx_cmd_query_flow_group_mb_out *)
	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
	dump = (uint8_t *)mbout;
	for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
		printf("%.2x ", dump[i]);
		if (i % 16 == 15)
			printf("\n");
	}
	dump = (uint8_t *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1)));
	for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
		printf("%.2x ", dump[i]);
		if (i % 16 == 15)
			printf("\n");
	}

free:
	mcx_cq_mboxes_free(sc, &mxm);
	return (error);
}

static int
mcx_dump_counters(struct mcx_softc *sc)
{
	struct mcx_dmamem mxm;
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_query_vport_counters_in *in;
	struct mcx_cmd_query_vport_counters_mb_in *mbin;
	struct mcx_cmd_query_vport_counters_out *out;
	struct mcx_nic_vport_counters *counters;
	int error, token;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
	    sizeof(*out) + sizeof(*counters), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_QUERY_VPORT_COUNTERS);
	in->cmd_op_mod = htobe16(0);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
	    &cqe->cq_output_ptr, token) != 0) {
		printf(", unable to allocate "
		    "query nic vport counters mailboxen\n");
		return (-1);
	}
	cqe->cq_input_ptr = cqe->cq_output_ptr;

	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
	mbin->cmd_clear = 0x80;

	mcx_cmdq_mboxes_sign(&mxm, 1);
	mcx_cmdq_post(sc, cqe, 0);

	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: query nic vport counters timeout\n", DEVNAME(sc));
		goto free;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: query nic vport counters command corrupt\n",
		    DEVNAME(sc));
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: query nic vport counters failed (%x, %x)\n",
		    DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

	counters = (struct mcx_nic_vport_counters *)
	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
	if (counters->rx_bcast.packets + counters->tx_bcast.packets +
	    counters->rx_ucast.packets + counters->tx_ucast.packets +
	    counters->rx_err.packets + counters->tx_err.packets)
		printf("%s: err %llx/%llx uc %llx/%llx bc %llx/%llx\n",
		    DEVNAME(sc),
		    betoh64(counters->tx_err.packets),
		    betoh64(counters->rx_err.packets),
		    betoh64(counters->tx_ucast.packets),
		    betoh64(counters->rx_ucast.packets),
		    betoh64(counters->tx_bcast.packets),
		    betoh64(counters->rx_bcast.packets));
free:
	mcx_dmamem_free(sc, &mxm);

	return (error);
}

static int
mcx_dump_flow_counter(struct mcx_softc *sc, int index, const char *what)
{
	struct mcx_dmamem mxm;
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_query_flow_counter_in *in;
	struct mcx_cmd_query_flow_counter_mb_in *mbin;
	struct mcx_cmd_query_flow_counter_out *out;
	struct mcx_counter *counters;
	int error, token;

	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
	token = mcx_cmdq_token(sc);
	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out) +
	    sizeof(*counters), token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_COUNTER);
	in->cmd_op_mod = htobe16(0);

	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
	    &cqe->cq_output_ptr, token) != 0) {
		printf(", unable to allocate query flow counter mailboxen\n");
		return (-1);
	}
	cqe->cq_input_ptr = cqe->cq_output_ptr;
	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
	mbin->cmd_flow_counter_id = htobe16(sc->sc_flow_counter_id[index]);
	mbin->cmd_clear = 0x80;

	mcx_cmdq_mboxes_sign(&mxm, 1);
	mcx_cmdq_post(sc, cqe, 0);

	error = mcx_cmdq_poll(sc, cqe, 1000);
	if (error != 0) {
		printf("%s: query flow counter timeout\n", DEVNAME(sc));
		goto free;
	}
	if (mcx_cmdq_verify(cqe) != 0) {
		printf("%s: query flow counter command corrupt\n", DEVNAME(sc));
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	if (out->cmd_status != MCX_CQ_STATUS_OK) {
		printf("%s: query flow counter failed (%x, %x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

	counters = (struct mcx_counter *)
	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
	if (counters->packets)
		printf("%s: %s inflow %llx\n", DEVNAME(sc), what,
		    betoh64(counters->packets));
free:
	mcx_dmamem_free(sc, &mxm);

	return (error);
}

#endif

#if NKSTAT > 0

int
mcx_query_rq(struct mcx_softc *sc, struct mcx_rx *rx, struct mcx_rq_ctx *rq_ctx)
{
	struct mcx_dmamem mxm;
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_query_rq_in *in;
	struct mcx_cmd_query_rq_out *out;
	struct mcx_cmd_query_rq_mb_out *mbout;
	uint8_t token = mcx_cmdq_token(sc);
	int error;

	cqe = mcx_get_cmdq_entry(sc, MCX_CMDQ_SLOT_KSTAT);
	if (cqe == NULL)
		return (-1);

	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16,
	    token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_QUERY_RQ);
	in->cmd_op_mod = htobe16(0);
	in->cmd_rqn = htobe32(rx->rx_rqn);

	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
	    &cqe->cq_output_ptr, token) != 0) {
		printf("%s: unable to allocate query rq mailboxes\n", DEVNAME(sc));
		return (-1);
	}

	mcx_cmdq_mboxes_sign(&mxm, 1);

	error = mcx_cmdq_exec(sc, cqe, MCX_CMDQ_SLOT_KSTAT, 1000);
	if (error != 0) {
		printf("%s: query rq timeout\n", DEVNAME(sc));
		goto free;
	}
	error = mcx_cmdq_verify(cqe);
	if (error != 0) {
		printf("%s: query rq reply corrupt\n", DEVNAME(sc));
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	switch (out->cmd_status) {
	case MCX_CQ_STATUS_OK:
		break;
	default:
		printf("%s: query rq failed (%x/%x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

        mbout = (struct mcx_cmd_query_rq_mb_out *)
	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
	memcpy(rq_ctx, &mbout->cmd_ctx, sizeof(*rq_ctx));

free:
	mcx_cq_mboxes_free(sc, &mxm);
	return (error);
}

int
mcx_query_sq(struct mcx_softc *sc, struct mcx_tx *tx, struct mcx_sq_ctx *sq_ctx)
{
	struct mcx_dmamem mxm;
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_query_sq_in *in;
	struct mcx_cmd_query_sq_out *out;
	struct mcx_cmd_query_sq_mb_out *mbout;
	uint8_t token = mcx_cmdq_token(sc);
	int error;

	cqe = mcx_get_cmdq_entry(sc, MCX_CMDQ_SLOT_KSTAT);
	if (cqe == NULL)
		return (-1);

	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16,
	    token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_QUERY_SQ);
	in->cmd_op_mod = htobe16(0);
	in->cmd_sqn = htobe32(tx->tx_sqn);

	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
	    &cqe->cq_output_ptr, token) != 0) {
		printf("%s: unable to allocate query sq mailboxes\n", DEVNAME(sc));
		return (-1);
	}

	mcx_cmdq_mboxes_sign(&mxm, 1);

	error = mcx_cmdq_exec(sc, cqe, MCX_CMDQ_SLOT_KSTAT, 1000);
	if (error != 0) {
		printf("%s: query sq timeout\n", DEVNAME(sc));
		goto free;
	}
	error = mcx_cmdq_verify(cqe);
	if (error != 0) {
		printf("%s: query sq reply corrupt\n", DEVNAME(sc));
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	switch (out->cmd_status) {
	case MCX_CQ_STATUS_OK:
		break;
	default:
		printf("%s: query sq failed (%x/%x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

        mbout = (struct mcx_cmd_query_sq_mb_out *)
	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
	memcpy(sq_ctx, &mbout->cmd_ctx, sizeof(*sq_ctx));

free:
	mcx_cq_mboxes_free(sc, &mxm);
	return (error);
}

int
mcx_query_cq(struct mcx_softc *sc, struct mcx_cq *cq, struct mcx_cq_ctx *cq_ctx)
{
	struct mcx_dmamem mxm;
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_query_cq_in *in;
	struct mcx_cmd_query_cq_out *out;
	struct mcx_cq_ctx *ctx;
	uint8_t token = mcx_cmdq_token(sc);
	int error;

	cqe = mcx_get_cmdq_entry(sc, MCX_CMDQ_SLOT_KSTAT);
	if (cqe == NULL)
		return (-1);

	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx) + 16,
	    token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_QUERY_CQ);
	in->cmd_op_mod = htobe16(0);
	in->cmd_cqn = htobe32(cq->cq_n);

	CTASSERT(sizeof(*ctx) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
	    &cqe->cq_output_ptr, token) != 0) {
		printf("%s: unable to allocate query cq mailboxes\n", DEVNAME(sc));
		return (-1);
	}

	mcx_cmdq_mboxes_sign(&mxm, 1);

	error = mcx_cmdq_exec(sc, cqe, MCX_CMDQ_SLOT_KSTAT, 1000);
	if (error != 0) {
		printf("%s: query cq timeout\n", DEVNAME(sc));
		goto free;
	}
	error = mcx_cmdq_verify(cqe);
	if (error != 0) {
		printf("%s: query cq reply corrupt\n", DEVNAME(sc));
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	switch (out->cmd_status) {
	case MCX_CQ_STATUS_OK:
		break;
	default:
		printf("%s: query cq failed (%x/%x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

        ctx = (struct mcx_cq_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
	memcpy(cq_ctx, ctx, sizeof(*cq_ctx));
free:
	mcx_cq_mboxes_free(sc, &mxm);
	return (error);
}

int
mcx_query_eq(struct mcx_softc *sc, struct mcx_eq *eq, struct mcx_eq_ctx *eq_ctx)
{
	struct mcx_dmamem mxm;
	struct mcx_cmdq_entry *cqe;
	struct mcx_cmd_query_eq_in *in;
	struct mcx_cmd_query_eq_out *out;
	struct mcx_eq_ctx *ctx;
	uint8_t token = mcx_cmdq_token(sc);
	int error;

	cqe = mcx_get_cmdq_entry(sc, MCX_CMDQ_SLOT_KSTAT);
	if (cqe == NULL)
		return (-1);

	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx) + 16,
	    token);

	in = mcx_cmdq_in(cqe);
	in->cmd_opcode = htobe16(MCX_CMD_QUERY_EQ);
	in->cmd_op_mod = htobe16(0);
	in->cmd_eqn = htobe32(eq->eq_n);

	CTASSERT(sizeof(*ctx) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
	    &cqe->cq_output_ptr, token) != 0) {
		printf("%s: unable to allocate query eq mailboxes\n", DEVNAME(sc));
		return (-1);
	}

	mcx_cmdq_mboxes_sign(&mxm, 1);

	error = mcx_cmdq_exec(sc, cqe, MCX_CMDQ_SLOT_KSTAT, 1000);
	if (error != 0) {
		printf("%s: query eq timeout\n", DEVNAME(sc));
		goto free;
	}
	error = mcx_cmdq_verify(cqe);
	if (error != 0) {
		printf("%s: query eq reply corrupt\n", DEVNAME(sc));
		goto free;
	}

	out = mcx_cmdq_out(cqe);
	switch (out->cmd_status) {
	case MCX_CQ_STATUS_OK:
		break;
	default:
		printf("%s: query eq failed (%x/%x)\n", DEVNAME(sc),
		    out->cmd_status, betoh32(out->cmd_syndrome));
		error = -1;
		goto free;
	}

        ctx = (struct mcx_eq_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
	memcpy(eq_ctx, ctx, sizeof(*eq_ctx));
free:
	mcx_cq_mboxes_free(sc, &mxm);
	return (error);
}

#endif /* NKSTAT > 0 */

static inline unsigned int
mcx_rx_fill_slots(struct mcx_softc *sc, struct mcx_rx *rx, uint nslots)
{
	struct mcx_rq_entry *ring, *rqe;
	struct mcx_slot *ms;
	struct mbuf *m;
	uint slot, p, fills;

	ring = MCX_DMA_KVA(&rx->rx_rq_mem);
	p = rx->rx_prod;

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
	    0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE);

	for (fills = 0; fills < nslots; fills++) {
		slot = p % (1 << MCX_LOG_RQ_SIZE);

		ms = &rx->rx_slots[slot];
		rqe = &ring[slot];

		m = MCLGETL(NULL, M_DONTWAIT, sc->sc_rxbufsz);
		if (m == NULL)
			break;

		m->m_data += (m->m_ext.ext_size - sc->sc_rxbufsz);
		m->m_data += ETHER_ALIGN;
		m->m_len = m->m_pkthdr.len = sc->sc_hardmtu;

		if (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
		    BUS_DMA_NOWAIT) != 0) {
			m_freem(m);
			break;
		}
		ms->ms_m = m;

		htobem32(&rqe->rqe_byte_count, ms->ms_map->dm_segs[0].ds_len);
		htobem64(&rqe->rqe_addr, ms->ms_map->dm_segs[0].ds_addr);
		htobem32(&rqe->rqe_lkey, sc->sc_lkey);

		p++;
	}

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
	    0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_PREWRITE);

	rx->rx_prod = p;

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
	    rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
	htobem32(MCX_DMA_OFF(&sc->sc_doorbell_mem, rx->rx_doorbell),
	    p & MCX_WQ_DOORBELL_MASK);
	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
	    rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);

	return (nslots - fills);
}

int
mcx_rx_fill(struct mcx_softc *sc, struct mcx_rx *rx)
{
	u_int slots;

	slots = if_rxr_get(&rx->rx_rxr, (1 << MCX_LOG_RQ_SIZE));
	if (slots == 0)
		return (1);

	slots = mcx_rx_fill_slots(sc, rx, slots);
	if_rxr_put(&rx->rx_rxr, slots);
	return (0);
}

void
mcx_refill(void *xrx)
{
	struct mcx_rx *rx = xrx;
	struct mcx_softc *sc = rx->rx_softc;

	mcx_rx_fill(sc, rx);

	if (if_rxr_inuse(&rx->rx_rxr) == 0)
		timeout_add(&rx->rx_refill, 1);
}

static int
mcx_process_txeof(struct mcx_softc *sc, struct mcx_tx *tx,
    struct mcx_cq_entry *cqe)
{
	struct mcx_slot *ms;
	bus_dmamap_t map;
	int slot, slots;

	slot = betoh16(cqe->cq_wqe_count) % (1 << MCX_LOG_SQ_SIZE);

	ms = &tx->tx_slots[slot];
	map = ms->ms_map;
	bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
	    BUS_DMASYNC_POSTWRITE);

	slots = 1;
	if (map->dm_nsegs > 1)
		slots += (map->dm_nsegs+2) / MCX_SQ_SEGS_PER_SLOT;

	bus_dmamap_unload(sc->sc_dmat, map);
	m_freem(ms->ms_m);
	ms->ms_m = NULL;

	return (slots);
}

static void
mcx_calibrate_first(struct mcx_softc *sc)
{
	struct mcx_calibration *c = &sc->sc_calibration[0];
	int s;

	sc->sc_calibration_gen = 0;

	s = splhigh(); /* crit_enter? */
	c->c_ubase = nsecuptime();
	c->c_tbase = mcx_timer(sc);
	splx(s);
	c->c_ratio = 0;

#ifdef notyet
	timeout_add_sec(&sc->sc_calibrate, MCX_CALIBRATE_FIRST);
#endif
}

#define MCX_TIMESTAMP_SHIFT 24

static void
mcx_calibrate(void *arg)
{
	struct mcx_softc *sc = arg;
	struct mcx_calibration *nc, *pc;
	uint64_t udiff, tdiff;
	unsigned int gen;
	int s;

	if (!ISSET(sc->sc_ac.ac_if.if_flags, IFF_RUNNING))
		return;

	timeout_add_sec(&sc->sc_calibrate, MCX_CALIBRATE_NORMAL);

	gen = sc->sc_calibration_gen;
	pc = &sc->sc_calibration[gen % nitems(sc->sc_calibration)];
	gen++;
	nc = &sc->sc_calibration[gen % nitems(sc->sc_calibration)];

	nc->c_uptime = pc->c_ubase;
	nc->c_timestamp = pc->c_tbase;

	s = splhigh(); /* crit_enter? */
	nc->c_ubase = nsecuptime();
	nc->c_tbase = mcx_timer(sc);
	splx(s);

	udiff = nc->c_ubase - nc->c_uptime;
	tdiff = nc->c_tbase - nc->c_timestamp;

	/*
	 * udiff is the wall clock time between calibration ticks,
	 * which should be 32 seconds or 32 billion nanoseconds. if
	 * we squint, 1 billion nanoseconds is kind of like a 32 bit
	 * number, so 32 billion should still have a lot of high bits
	 * spare. we use this space by shifting the nanoseconds up
	 * 24 bits so we have a nice big number to divide by the
	 * number of mcx timer ticks.
	 */
	nc->c_ratio = (udiff << MCX_TIMESTAMP_SHIFT) / tdiff;

	membar_producer();
	sc->sc_calibration_gen = gen;
}

static int
mcx_process_rx(struct mcx_softc *sc, struct mcx_rx *rx,
    struct mcx_cq_entry *cqe, struct mbuf_list *ml,
    const struct mcx_calibration *c)
{
	struct mcx_slot *ms;
	struct mbuf *m;
	uint32_t flags, len;
	int slot;

	len = bemtoh32(&cqe->cq_byte_cnt);
	slot = betoh16(cqe->cq_wqe_count) % (1 << MCX_LOG_RQ_SIZE);

	ms = &rx->rx_slots[slot];
	bus_dmamap_sync(sc->sc_dmat, ms->ms_map, 0, len, BUS_DMASYNC_POSTREAD);
	bus_dmamap_unload(sc->sc_dmat, ms->ms_map);

	m = ms->ms_m;
	ms->ms_m = NULL;

	m->m_pkthdr.len = m->m_len = len;

	if (cqe->cq_rx_hash_type) {
		m->m_pkthdr.ph_flowid = betoh32(cqe->cq_rx_hash);
		m->m_pkthdr.csum_flags |= M_FLOWID;
	}

	flags = bemtoh32(&cqe->cq_flags);
	if (flags & MCX_CQ_ENTRY_FLAGS_L3_OK)
		m->m_pkthdr.csum_flags |= M_IPV4_CSUM_IN_OK;
	if (flags & MCX_CQ_ENTRY_FLAGS_L4_OK)
		m->m_pkthdr.csum_flags |= M_TCP_CSUM_IN_OK |
		    M_UDP_CSUM_IN_OK;
#if NVLAN > 0
	if (flags & MCX_CQ_ENTRY_FLAGS_CV) {
		m->m_pkthdr.ether_vtag = (flags &
		    MCX_CQ_ENTRY_FLAGS_VLAN_MASK);
		m->m_flags |= M_VLANTAG;
	}
#endif

#ifdef notyet
	if (ISSET(sc->sc_ac.ac_if.if_flags, IFF_LINK0) && c->c_ratio) {
		uint64_t t = bemtoh64(&cqe->cq_timestamp);
		t -= c->c_timestamp;
		t *= c->c_ratio;
		t >>= MCX_TIMESTAMP_SHIFT;
		t += c->c_uptime;

		m->m_pkthdr.ph_timestamp = t;
		SET(m->m_pkthdr.csum_flags, M_TIMESTAMP);
	}
#endif

	ml_enqueue(ml, m);

	return (1);
}

static struct mcx_cq_entry *
mcx_next_cq_entry(struct mcx_softc *sc, struct mcx_cq *cq)
{
	struct mcx_cq_entry *cqe;
	int next;

	cqe = (struct mcx_cq_entry *)MCX_DMA_KVA(&cq->cq_mem);
	next = cq->cq_cons % (1 << MCX_LOG_CQ_SIZE);

	if ((cqe[next].cq_opcode_owner & MCX_CQ_ENTRY_FLAG_OWNER) ==
	    ((cq->cq_cons >> MCX_LOG_CQ_SIZE) & 1)) {
		return (&cqe[next]);
	}

	return (NULL);
}

static void
mcx_arm_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar)
{
	struct mcx_cq_doorbell *db;
	bus_size_t offset;
	uint32_t val;
	uint64_t uval;

	val = ((cq->cq_count) & 3) << MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT;
	val |= (cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK);

	db = MCX_DMA_OFF(&sc->sc_doorbell_mem, cq->cq_doorbell);

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
	    cq->cq_doorbell, sizeof(*db), BUS_DMASYNC_POSTWRITE);

	htobem32(&db->db_update_ci, cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK);
	htobem32(&db->db_arm_ci, val);

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
	    cq->cq_doorbell, sizeof(*db), BUS_DMASYNC_PREWRITE);

	offset = (MCX_PAGE_SIZE * uar) + MCX_UAR_CQ_DOORBELL;

	uval = (uint64_t)val << 32;
	uval |= cq->cq_n;

	bus_space_write_raw_8(sc->sc_memt, sc->sc_memh, offset, htobe64(uval));
	mcx_bar(sc, offset, sizeof(uval), BUS_SPACE_BARRIER_WRITE);
}

void
mcx_process_cq(struct mcx_softc *sc, struct mcx_queues *q, struct mcx_cq *cq)
{
	struct mcx_rx *rx = &q->q_rx;
	struct mcx_tx *tx = &q->q_tx;
	const struct mcx_calibration *c;
	unsigned int gen;
	struct mcx_cq_entry *cqe;
	uint8_t *cqp;
	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
	int rxfree, txfree;

	gen = sc->sc_calibration_gen;
	membar_consumer();
	c = &sc->sc_calibration[gen % nitems(sc->sc_calibration)];

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
	    0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD);

	rxfree = 0;
	txfree = 0;
	while ((cqe = mcx_next_cq_entry(sc, cq))) {
		uint8_t opcode;
		opcode = (cqe->cq_opcode_owner >> MCX_CQ_ENTRY_OPCODE_SHIFT);
		switch (opcode) {
		case MCX_CQ_ENTRY_OPCODE_REQ:
			txfree += mcx_process_txeof(sc, tx, cqe);
			break;
		case MCX_CQ_ENTRY_OPCODE_SEND:
			rxfree += mcx_process_rx(sc, rx, cqe, &ml, c);
			break;
		case MCX_CQ_ENTRY_OPCODE_REQ_ERR:
		case MCX_CQ_ENTRY_OPCODE_SEND_ERR:
			cqp = (uint8_t *)cqe;
			/* printf("%s: cq completion error: %x\n",
			    DEVNAME(sc), cqp[0x37]); */
			break;

		default:
			/* printf("%s: cq completion opcode %x??\n",
			    DEVNAME(sc), opcode); */
			break;
		}

		cq->cq_cons++;
	}

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
	    0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_PREREAD);

	if (rxfree > 0) {
		if_rxr_put(&rx->rx_rxr, rxfree);
		if (ifiq_input(rx->rx_ifiq, &ml))
			if_rxr_livelocked(&rx->rx_rxr);

		mcx_rx_fill(sc, rx);
		if (if_rxr_inuse(&rx->rx_rxr) == 0)
			timeout_add(&rx->rx_refill, 1);
	}

	cq->cq_count++;
	mcx_arm_cq(sc, cq, q->q_uar);

	if (txfree > 0) {
		tx->tx_cons += txfree;
		if (ifq_is_oactive(tx->tx_ifq))
			ifq_restart(tx->tx_ifq);
	}
}


static void
mcx_arm_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar)
{
	bus_size_t offset;
	uint32_t val;

	offset = (MCX_PAGE_SIZE * uar) + MCX_UAR_EQ_DOORBELL_ARM;
	val = (eq->eq_n << 24) | (eq->eq_cons & 0xffffff);

	mcx_wr(sc, offset, val);
	mcx_bar(sc, offset, sizeof(val), BUS_SPACE_BARRIER_WRITE);
}

static struct mcx_eq_entry *
mcx_next_eq_entry(struct mcx_softc *sc, struct mcx_eq *eq)
{
	struct mcx_eq_entry *eqe;
	int next;

	eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem);
	next = eq->eq_cons % (1 << MCX_LOG_EQ_SIZE);
	if ((eqe[next].eq_owner & 1) ==
	    ((eq->eq_cons >> MCX_LOG_EQ_SIZE) & 1)) {
		eq->eq_cons++;
		return (&eqe[next]);
	}
	return (NULL);
}

int
mcx_admin_intr(void *xsc)
{
	struct mcx_softc *sc = (struct mcx_softc *)xsc;
	struct mcx_eq *eq = &sc->sc_admin_eq;
	struct mcx_eq_entry *eqe;

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
	    0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD);

	while ((eqe = mcx_next_eq_entry(sc, eq)) != NULL) {
		switch (eqe->eq_event_type) {
		case MCX_EVENT_TYPE_LAST_WQE:
			/* printf("%s: last wqe reached?\n", DEVNAME(sc)); */
			break;

		case MCX_EVENT_TYPE_CQ_ERROR:
			/* printf("%s: cq error\n", DEVNAME(sc)); */
			break;

		case MCX_EVENT_TYPE_CMD_COMPLETION:
			mtx_enter(&sc->sc_cmdq_mtx);
			wakeup(&sc->sc_cmdq_token);
			mtx_leave(&sc->sc_cmdq_mtx);
			break;

		case MCX_EVENT_TYPE_PORT_CHANGE:
			task_add(systq, &sc->sc_port_change);
			break;

		default:
			/* printf("%s: something happened\n", DEVNAME(sc)); */
			break;
		}
	}

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
	    0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD);

	mcx_arm_eq(sc, eq, sc->sc_uar);

	return (1);
}

int
mcx_cq_intr(void *xq)
{
	struct mcx_queues *q = (struct mcx_queues *)xq;
	struct mcx_softc *sc = q->q_sc;
	struct mcx_eq *eq = &q->q_eq;
	struct mcx_eq_entry *eqe;
	int cqn;

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
	    0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD);

	while ((eqe = mcx_next_eq_entry(sc, eq)) != NULL) {
		switch (eqe->eq_event_type) {
		case MCX_EVENT_TYPE_COMPLETION:
			cqn = betoh32(eqe->eq_event_data[6]);
			if (cqn == q->q_cq.cq_n)
				mcx_process_cq(sc, q, &q->q_cq);
			break;
		}
	}

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
	    0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD);

	mcx_arm_eq(sc, eq, q->q_uar);

	return (1);
}

static void
mcx_free_slots(struct mcx_softc *sc, struct mcx_slot *slots, int allocated,
    int total)
{
	struct mcx_slot *ms;

	int i = allocated;
	while (i-- > 0) {
		ms = &slots[i];
		bus_dmamap_destroy(sc->sc_dmat, ms->ms_map);
		if (ms->ms_m != NULL)
			m_freem(ms->ms_m);
	}
	free(slots, M_DEVBUF, total * sizeof(*ms));
}

static int
mcx_queue_up(struct mcx_softc *sc, struct mcx_queues *q)
{
	struct mcx_rx *rx;
	struct mcx_tx *tx;
	struct mcx_slot *ms;
	int i;

	rx = &q->q_rx;
	rx->rx_slots = mallocarray(sizeof(*ms), (1 << MCX_LOG_RQ_SIZE),
	    M_DEVBUF, M_WAITOK | M_ZERO);
	if (rx->rx_slots == NULL) {
		printf("%s: failed to allocate rx slots\n", DEVNAME(sc));
		return ENOMEM;
	}

	for (i = 0; i < (1 << MCX_LOG_RQ_SIZE); i++) {
		ms = &rx->rx_slots[i];
		if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu, 1,
		    sc->sc_hardmtu, 0,
		    BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT,
		    &ms->ms_map) != 0) {
			printf("%s: failed to allocate rx dma maps\n",
			    DEVNAME(sc));
			goto destroy_rx_slots;
		}
	}

	tx = &q->q_tx;
	tx->tx_slots = mallocarray(sizeof(*ms), (1 << MCX_LOG_SQ_SIZE),
	    M_DEVBUF, M_WAITOK | M_ZERO);
	if (tx->tx_slots == NULL) {
		printf("%s: failed to allocate tx slots\n", DEVNAME(sc));
		goto destroy_rx_slots;
	}

	for (i = 0; i < (1 << MCX_LOG_SQ_SIZE); i++) {
		ms = &tx->tx_slots[i];
		if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu,
		    MCX_SQ_MAX_SEGMENTS, sc->sc_hardmtu, 0,
		    BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT,
		    &ms->ms_map) != 0) {
			printf("%s: failed to allocate tx dma maps\n",
			    DEVNAME(sc));
			goto destroy_tx_slots;
		}
	}

	if (mcx_create_cq(sc, &q->q_cq, q->q_uar, q->q_index,
	    q->q_eq.eq_n) != 0)
		goto destroy_tx_slots;

	if (mcx_create_sq(sc, tx, q->q_uar, q->q_index, q->q_cq.cq_n)
	    != 0)
		goto destroy_cq;

	if (mcx_create_rq(sc, rx, q->q_index, q->q_cq.cq_n) != 0)
		goto destroy_sq;

	return 0;

destroy_sq:
	mcx_destroy_sq(sc, tx);
destroy_cq:
	mcx_destroy_cq(sc, &q->q_cq);
destroy_tx_slots:
	mcx_free_slots(sc, tx->tx_slots, i, (1 << MCX_LOG_SQ_SIZE));
	tx->tx_slots = NULL;

	i = (1 << MCX_LOG_RQ_SIZE);
destroy_rx_slots:
	mcx_free_slots(sc, rx->rx_slots, i, (1 << MCX_LOG_RQ_SIZE));
	rx->rx_slots = NULL;
	return ENOMEM;
}

static int
mcx_rss_group_entry_count(struct mcx_softc *sc, int group)
{
	int i;
	int count;

	count = 0;
	for (i = 0; i < nitems(mcx_rss_config); i++) {
		if (mcx_rss_config[i].flow_group == group)
			count++;
	}

	return count;
}

static int
mcx_up(struct mcx_softc *sc)
{
	struct ifnet *ifp = &sc->sc_ac.ac_if;
	struct mcx_rx *rx;
	struct mcx_tx *tx;
	int i, start, count, flow_group, flow_index;
	struct mcx_flow_match match_crit;
	struct mcx_rss_rule *rss;
	uint32_t dest;
	int rqns[MCX_MAX_QUEUES];

	if (mcx_create_tis(sc, &sc->sc_tis) != 0)
		goto down;

	for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) {
		if (mcx_queue_up(sc, &sc->sc_queues[i]) != 0) {
			goto down;
		}
	}

	/* RSS flow table and flow groups */
	if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 1,
	    &sc->sc_rss_flow_table_id) != 0)
		goto down;

	dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE |
	    sc->sc_rss_flow_table_id;

	/* L4 RSS flow group (v4/v6 tcp/udp, no fragments) */
	memset(&match_crit, 0, sizeof(match_crit));
	match_crit.mc_ethertype = 0xffff;
	match_crit.mc_ip_proto = 0xff;
	match_crit.mc_vlan_flags = MCX_FLOW_MATCH_IP_FRAG;
	start = 0;
	count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L4);
	if (count != 0) {
		if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id,
		    MCX_FLOW_GROUP_RSS_L4, start, count,
		    MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
			goto down;
		start += count;
	}

	/* L3 RSS flow group (v4/v6, including fragments) */
	memset(&match_crit, 0, sizeof(match_crit));
	match_crit.mc_ethertype = 0xffff;
	count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L3);
	if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id,
	    MCX_FLOW_GROUP_RSS_L3, start, count,
	    MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
		goto down;
	start += count;

	/* non-RSS flow group */
	count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_NONE);
	memset(&match_crit, 0, sizeof(match_crit));
	if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id,
	    MCX_FLOW_GROUP_RSS_NONE, start, count, 0, &match_crit) != 0)
		goto down;

	/* Root flow table, matching packets based on mac address */
	if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 0,
	    &sc->sc_mac_flow_table_id) != 0)
		goto down;

	/* promisc flow group */
	start = 0;
	memset(&match_crit, 0, sizeof(match_crit));
	if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id,
	    MCX_FLOW_GROUP_PROMISC, start, 1, 0, &match_crit) != 0)
		goto down;
	sc->sc_promisc_flow_enabled = 0;
	start++;

	/* all multicast flow group */
	match_crit.mc_dest_mac[0] = 0x01;
	if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id,
	    MCX_FLOW_GROUP_ALLMULTI, start, 1,
	    MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
		goto down;
	sc->sc_allmulti_flow_enabled = 0;
	start++;

	/* mac address matching flow group */
	memset(&match_crit.mc_dest_mac, 0xff, sizeof(match_crit.mc_dest_mac));
	if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id,
	    MCX_FLOW_GROUP_MAC, start, (1 << MCX_LOG_FLOW_TABLE_SIZE) - start,
	    MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
		goto down;

	/* flow table entries for unicast and broadcast */
	start = 0;
	if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start,
	    sc->sc_ac.ac_enaddr, dest) != 0)
		goto down;
	start++;

	if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start,
	    etherbroadcastaddr, dest) != 0)
		goto down;
	start++;

	/* multicast entries go after that */
	sc->sc_mcast_flow_base = start;

	/* re-add any existing multicast flows */
	for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
		if (sc->sc_mcast_flows[i][0] != 0) {
			mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC,
			    sc->sc_mcast_flow_base + i,
			    sc->sc_mcast_flows[i], dest);
		}
	}

	if (mcx_set_flow_table_root(sc, sc->sc_mac_flow_table_id) != 0)
		goto down;

	/*
	 * the RQT can be any size as long as it's a power of two.
	 * since we also restrict the number of queues to a power of two,
	 * we can just put each rx queue in once.
	 */
	for (i = 0; i < intrmap_count(sc->sc_intrmap); i++)
		rqns[i] = sc->sc_queues[i].q_rx.rx_rqn;

	if (mcx_create_rqt(sc, intrmap_count(sc->sc_intrmap), rqns,
	    &sc->sc_rqt) != 0)
		goto down;

	start = 0;
	flow_index = 0;
	flow_group = -1;
	for (i = 0; i < nitems(mcx_rss_config); i++) {
		rss = &mcx_rss_config[i];
		if (rss->flow_group != flow_group) {
			flow_group = rss->flow_group;
			flow_index = 0;
		}

		if (rss->hash_sel == 0) {
			if (mcx_create_tir_direct(sc, &sc->sc_queues[0].q_rx,
			    &sc->sc_tir[i]) != 0)
				goto down;
		} else {
			if (mcx_create_tir_indirect(sc, sc->sc_rqt,
			    rss->hash_sel, &sc->sc_tir[i]) != 0)
				goto down;
		}

		if (mcx_set_flow_table_entry_proto(sc, flow_group,
		    flow_index, rss->ethertype, rss->ip_proto,
		    MCX_FLOW_CONTEXT_DEST_TYPE_TIR | sc->sc_tir[i]) != 0)
			goto down;
		flow_index++;
	}

	for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) {
		struct mcx_queues *q = &sc->sc_queues[i];
		rx = &q->q_rx;
		tx = &q->q_tx;

		/* start the queues */
		if (mcx_ready_sq(sc, tx) != 0)
			goto down;

		if (mcx_ready_rq(sc, rx) != 0)
			goto down;

		if_rxr_init(&rx->rx_rxr, 1, (1 << MCX_LOG_RQ_SIZE));
		rx->rx_prod = 0;
		mcx_rx_fill(sc, rx);

		tx->tx_cons = 0;
		tx->tx_prod = 0;
		ifq_clr_oactive(tx->tx_ifq);
	}

	mcx_calibrate_first(sc);

	SET(ifp->if_flags, IFF_RUNNING);

	return ENETRESET;
down:
	mcx_down(sc);
	return ENOMEM;
}

static void
mcx_down(struct mcx_softc *sc)
{
	struct ifnet *ifp = &sc->sc_ac.ac_if;
	struct mcx_rss_rule *rss;
	int group, i, flow_group, flow_index;

	CLR(ifp->if_flags, IFF_RUNNING);

	/*
	 * delete flow table entries first, so no packets can arrive
	 * after the barriers
	 */
	if (sc->sc_promisc_flow_enabled)
		mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0);
	if (sc->sc_allmulti_flow_enabled)
		mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0);
	mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 0);
	mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 1);
	for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
		if (sc->sc_mcast_flows[i][0] != 0) {
			mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC,
			    sc->sc_mcast_flow_base + i);
		}
	}

	flow_group = -1;
	flow_index = 0;
	for (i = 0; i < nitems(mcx_rss_config); i++) {
		rss = &mcx_rss_config[i];
		if (rss->flow_group != flow_group) {
			flow_group = rss->flow_group;
			flow_index = 0;
		}

		mcx_delete_flow_table_entry(sc, flow_group, flow_index);

		mcx_destroy_tir(sc, sc->sc_tir[i]);
		sc->sc_tir[i] = 0;

		flow_index++;
	}
	intr_barrier(sc->sc_ihc);
	for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) {
		struct ifqueue *ifq = sc->sc_queues[i].q_tx.tx_ifq;
		ifq_barrier(ifq);

		timeout_del_barrier(&sc->sc_queues[i].q_rx.rx_refill);

		intr_barrier(sc->sc_queues[i].q_ihc);
	}

	timeout_del_barrier(&sc->sc_calibrate);

	for (group = 0; group < MCX_NUM_FLOW_GROUPS; group++) {
		if (sc->sc_flow_group[group].g_id != -1)
			mcx_destroy_flow_group(sc, group);
	}

	if (sc->sc_mac_flow_table_id != -1) {
		mcx_destroy_flow_table(sc, sc->sc_mac_flow_table_id);
		sc->sc_mac_flow_table_id = -1;
	}
	if (sc->sc_rss_flow_table_id != -1) {
		mcx_destroy_flow_table(sc, sc->sc_rss_flow_table_id);
		sc->sc_rss_flow_table_id = -1;
	}
	if (sc->sc_rqt != -1) {
		mcx_destroy_rqt(sc, sc->sc_rqt);
		sc->sc_rqt = -1;
	}

	for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) {
		struct mcx_queues *q = &sc->sc_queues[i];
		struct mcx_rx *rx = &q->q_rx;
		struct mcx_tx *tx = &q->q_tx;
		struct mcx_cq *cq = &q->q_cq;

		if (rx->rx_rqn != 0)
			mcx_destroy_rq(sc, rx);

		if (tx->tx_sqn != 0)
			mcx_destroy_sq(sc, tx);

		if (tx->tx_slots != NULL) {
			mcx_free_slots(sc, tx->tx_slots,
			    (1 << MCX_LOG_SQ_SIZE), (1 << MCX_LOG_SQ_SIZE));
			tx->tx_slots = NULL;
		}
		if (rx->rx_slots != NULL) {
			mcx_free_slots(sc, rx->rx_slots,
			    (1 << MCX_LOG_RQ_SIZE), (1 << MCX_LOG_RQ_SIZE));
			rx->rx_slots = NULL;
		}

		if (cq->cq_n != 0)
			mcx_destroy_cq(sc, cq);
	}
	if (sc->sc_tis != 0) {
		mcx_destroy_tis(sc, sc->sc_tis);
		sc->sc_tis = 0;
	}
}

static int
mcx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
{
	struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
	struct ifreq *ifr = (struct ifreq *)data;
	uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN];
	int s, i, error = 0;
	uint32_t dest;

	s = splnet();
	switch (cmd) {
	case SIOCSIFADDR:
		ifp->if_flags |= IFF_UP;
		/* FALLTHROUGH */

	case SIOCSIFFLAGS:
		if (ISSET(ifp->if_flags, IFF_UP)) {
			if (ISSET(ifp->if_flags, IFF_RUNNING))
				error = ENETRESET;
			else
				error = mcx_up(sc);
		} else {
			if (ISSET(ifp->if_flags, IFF_RUNNING))
				mcx_down(sc);
		}
		break;

	case SIOCGIFMEDIA:
	case SIOCSIFMEDIA:
		error = ifmedia_ioctl(ifp, ifr, &sc->sc_media, cmd);
		break;

	case SIOCGIFSFFPAGE:
		error = mcx_get_sffpage(ifp, (struct if_sffpage *)data);
		break;

	case SIOCGIFRXR:
		error = mcx_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data);
		break;

	case SIOCADDMULTI:
		if (ether_addmulti(ifr, &sc->sc_ac) == ENETRESET) {
			error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi);
			if (error != 0)
				break;

			dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE |
			    sc->sc_rss_flow_table_id;

			for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
				if (sc->sc_mcast_flows[i][0] == 0) {
					memcpy(sc->sc_mcast_flows[i], addrlo,
					    ETHER_ADDR_LEN);
					if (ISSET(ifp->if_flags, IFF_RUNNING)) {
						mcx_set_flow_table_entry_mac(sc,
						    MCX_FLOW_GROUP_MAC,
						    sc->sc_mcast_flow_base + i,
						    sc->sc_mcast_flows[i], dest);
					}
					break;
				}
			}

			if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) {
				if (i == MCX_NUM_MCAST_FLOWS) {
					SET(ifp->if_flags, IFF_ALLMULTI);
					sc->sc_extra_mcast++;
					error = ENETRESET;
				}

				if (sc->sc_ac.ac_multirangecnt > 0) {
					SET(ifp->if_flags, IFF_ALLMULTI);
					error = ENETRESET;
				}
			}
		}
		break;

	case SIOCDELMULTI:
		if (ether_delmulti(ifr, &sc->sc_ac) == ENETRESET) {
			error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi);
			if (error != 0)
				break;

			for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
				if (memcmp(sc->sc_mcast_flows[i], addrlo,
				    ETHER_ADDR_LEN) == 0) {
					if (ISSET(ifp->if_flags, IFF_RUNNING)) {
						mcx_delete_flow_table_entry(sc,
						    MCX_FLOW_GROUP_MAC,
						    sc->sc_mcast_flow_base + i);
					}
					sc->sc_mcast_flows[i][0] = 0;
					break;
				}
			}

			if (i == MCX_NUM_MCAST_FLOWS)
				sc->sc_extra_mcast--;

			if (ISSET(ifp->if_flags, IFF_ALLMULTI) &&
			    (sc->sc_extra_mcast == 0) &&
			    (sc->sc_ac.ac_multirangecnt == 0)) {
				CLR(ifp->if_flags, IFF_ALLMULTI);
				error = ENETRESET;
			}
		}
		break;

	default:
		error = ether_ioctl(ifp, &sc->sc_ac, cmd, data);
	}

	if (error == ENETRESET) {
		if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) ==
		    (IFF_UP | IFF_RUNNING))
			mcx_iff(sc);
		error = 0;
	}
	splx(s);

	return (error);
}

static int
mcx_get_sffpage(struct ifnet *ifp, struct if_sffpage *sff)
{
	struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
	struct mcx_reg_mcia mcia;
	struct mcx_reg_pmlp pmlp;
	int offset, error;

	rw_enter_write(&sc->sc_cmdq_ioctl_lk);

	/* get module number */
	memset(&pmlp, 0, sizeof(pmlp));
	pmlp.rp_local_port = 1;
	error = mcx_access_hca_reg(sc, MCX_REG_PMLP, MCX_REG_OP_READ, &pmlp,
	    sizeof(pmlp), MCX_CMDQ_SLOT_IOCTL);
	if (error != 0) {
		printf("%s: unable to get eeprom module number\n",
		    DEVNAME(sc));
		goto out;
	}

	for (offset = 0; offset < 256; offset += MCX_MCIA_EEPROM_BYTES) {
		memset(&mcia, 0, sizeof(mcia));
		mcia.rm_l = 0;
		mcia.rm_module = betoh32(pmlp.rp_lane0_mapping) &
		    MCX_PMLP_MODULE_NUM_MASK;
		mcia.rm_i2c_addr = sff->sff_addr / 2;	/* apparently */
		mcia.rm_page_num = sff->sff_page;
		mcia.rm_dev_addr = htobe16(offset);
		mcia.rm_size = htobe16(MCX_MCIA_EEPROM_BYTES);

		error = mcx_access_hca_reg(sc, MCX_REG_MCIA, MCX_REG_OP_READ,
		    &mcia, sizeof(mcia), MCX_CMDQ_SLOT_IOCTL);
		if (error != 0) {
			printf("%s: unable to read eeprom at %x\n",
			    DEVNAME(sc), offset);
			goto out;
		}

		memcpy(sff->sff_data + offset, mcia.rm_data,
		    MCX_MCIA_EEPROM_BYTES);
	}

 out:
	rw_exit_write(&sc->sc_cmdq_ioctl_lk);
	return (error);
}

static int
mcx_rxrinfo(struct mcx_softc *sc, struct if_rxrinfo *ifri)
{
	struct if_rxring_info *ifrs;
	unsigned int i;
	int error;

	ifrs = mallocarray(intrmap_count(sc->sc_intrmap), sizeof(*ifrs),
	    M_TEMP, M_WAITOK|M_ZERO|M_CANFAIL);
	if (ifrs == NULL)
		return (ENOMEM);

	for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) {
		struct mcx_rx *rx = &sc->sc_queues[i].q_rx;
		struct if_rxring_info *ifr = &ifrs[i];

		snprintf(ifr->ifr_name, sizeof(ifr->ifr_name), "%u", i);
		ifr->ifr_size = sc->sc_hardmtu;
		ifr->ifr_info = rx->rx_rxr;
	}

	error = if_rxr_info_ioctl(ifri, i, ifrs);
	free(ifrs, M_TEMP, i * sizeof(*ifrs));

	return (error);
}

int
mcx_load_mbuf(struct mcx_softc *sc, struct mcx_slot *ms, struct mbuf *m)
{
	switch (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
	    BUS_DMA_STREAMING | BUS_DMA_NOWAIT)) {
	case 0:
		break;

	case EFBIG:
		if (m_defrag(m, M_DONTWAIT) == 0 &&
		    bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
		    BUS_DMA_STREAMING | BUS_DMA_NOWAIT) == 0)
			break;

	default:
		return (1);
	}

	ms->ms_m = m;
	return (0);
}

static void
mcx_start(struct ifqueue *ifq)
{
	struct mcx_tx *tx = ifq->ifq_softc;
	struct ifnet *ifp = ifq->ifq_if;
	struct mcx_softc *sc = ifp->if_softc;
	struct mcx_sq_entry *sq, *sqe;
	struct mcx_sq_entry_seg *sqs;
	struct mcx_slot *ms;
	bus_dmamap_t map;
	struct mbuf *m;
	u_int idx, free, used;
	uint64_t *bf;
	uint32_t csum;
	size_t bf_base;
	int i, seg, nseg;

	bf_base = (tx->tx_uar * MCX_PAGE_SIZE) + MCX_UAR_BF;

	idx = tx->tx_prod % (1 << MCX_LOG_SQ_SIZE);
	free = (tx->tx_cons + (1 << MCX_LOG_SQ_SIZE)) - tx->tx_prod;

	used = 0;
	bf = NULL;

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
	    0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE);

	sq = (struct mcx_sq_entry *)MCX_DMA_KVA(&tx->tx_sq_mem);

	for (;;) {
		if (used + MCX_SQ_ENTRY_MAX_SLOTS >= free) {
			ifq_set_oactive(ifq);
			break;
		}

		m = ifq_dequeue(ifq);
		if (m == NULL) {
			break;
		}

		sqe = sq + idx;
		ms = &tx->tx_slots[idx];
		memset(sqe, 0, sizeof(*sqe));

		/* ctrl segment */
		sqe->sqe_opcode_index = htobe32(MCX_SQE_WQE_OPCODE_SEND |
		    ((tx->tx_prod & 0xffff) << MCX_SQE_WQE_INDEX_SHIFT));
		/* always generate a completion event */
		sqe->sqe_signature = htobe32(MCX_SQE_CE_CQE_ALWAYS);

		/* eth segment */
		csum = 0;
		if (m->m_pkthdr.csum_flags & M_IPV4_CSUM_OUT)
			csum |= MCX_SQE_L3_CSUM;
		if (m->m_pkthdr.csum_flags & (M_TCP_CSUM_OUT | M_UDP_CSUM_OUT))
			csum |= MCX_SQE_L4_CSUM;
		sqe->sqe_mss_csum = htobe32(csum);
		sqe->sqe_inline_header_size = htobe16(MCX_SQ_INLINE_SIZE);
#if NVLAN > 0
		if (m->m_flags & M_VLANTAG) {
			struct ether_vlan_header *evh;
			evh = (struct ether_vlan_header *)
			    &sqe->sqe_inline_headers;

			/* slightly cheaper vlan_inject() */
			m_copydata(m, 0, ETHER_HDR_LEN, evh);
			evh->evl_proto = evh->evl_encap_proto;
			evh->evl_encap_proto = htons(ETHERTYPE_VLAN);
			evh->evl_tag = htons(m->m_pkthdr.ether_vtag);

			m_adj(m, ETHER_HDR_LEN);
		} else
#endif
		{
			m_copydata(m, 0, MCX_SQ_INLINE_SIZE,
			    sqe->sqe_inline_headers);
			m_adj(m, MCX_SQ_INLINE_SIZE);
		}

		if (mcx_load_mbuf(sc, ms, m) != 0) {
			m_freem(m);
			ifp->if_oerrors++;
			continue;
		}
		bf = (uint64_t *)sqe;

#if NBPFILTER > 0
		if (ifp->if_bpf)
			bpf_mtap_hdr(ifp->if_bpf,
			    (caddr_t)sqe->sqe_inline_headers,
			    MCX_SQ_INLINE_SIZE, m, BPF_DIRECTION_OUT);
#endif
		map = ms->ms_map;
		bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
		    BUS_DMASYNC_PREWRITE);

		sqe->sqe_ds_sq_num =
		    htobe32((tx->tx_sqn << MCX_SQE_SQ_NUM_SHIFT) |
		    (map->dm_nsegs + 3));

		/* data segment - first wqe has one segment */
		sqs = sqe->sqe_segs;
		seg = 0;
		nseg = 1;
		for (i = 0; i < map->dm_nsegs; i++) {
			if (seg == nseg) {
				/* next slot */
				idx++;
				if (idx == (1 << MCX_LOG_SQ_SIZE))
					idx = 0;
				tx->tx_prod++;
				used++;

				sqs = (struct mcx_sq_entry_seg *)(sq + idx);
				seg = 0;
				nseg = MCX_SQ_SEGS_PER_SLOT;
			}
			sqs[seg].sqs_byte_count =
			    htobe32(map->dm_segs[i].ds_len);
			sqs[seg].sqs_lkey = htobe32(sc->sc_lkey);
			sqs[seg].sqs_addr = htobe64(map->dm_segs[i].ds_addr);
			seg++;
		}

		idx++;
		if (idx == (1 << MCX_LOG_SQ_SIZE))
			idx = 0;
		tx->tx_prod++;
		used++;
	}

	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
	    0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_PREWRITE);

	if (used) {
		bus_size_t blueflame;

		bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
		    tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
		htobem32(MCX_DMA_OFF(&sc->sc_doorbell_mem, tx->tx_doorbell),
		    tx->tx_prod & MCX_WQ_DOORBELL_MASK);
		bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
		    tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);

		/*
		 * write the first 64 bits of the last sqe we produced
		 * to the blue flame buffer
		 */

		blueflame = bf_base + tx->tx_bf_offset;
		bus_space_write_raw_8(sc->sc_memt, sc->sc_memh,
		    blueflame, *bf);
		mcx_bar(sc, blueflame, sizeof(*bf), BUS_SPACE_BARRIER_WRITE);

		/* next write goes to the other buffer */
		tx->tx_bf_offset ^= sc->sc_bf_size;
	}
}

static void
mcx_watchdog(struct ifnet *ifp)
{
}

static void
mcx_media_add_types(struct mcx_softc *sc)
{
	struct mcx_reg_ptys ptys;
	int i;
	uint32_t proto_cap;

	memset(&ptys, 0, sizeof(ptys));
	ptys.rp_local_port = 1;
	ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
	if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys,
	    sizeof(ptys), MCX_CMDQ_SLOT_POLL) != 0) {
		printf("%s: unable to read port type/speed\n", DEVNAME(sc));
		return;
	}

	proto_cap = betoh32(ptys.rp_eth_proto_cap);
	for (i = 0; i < nitems(mcx_eth_cap_map); i++) {
		const struct mcx_eth_proto_capability *cap;
		if (!ISSET(proto_cap, 1 << i))
			continue;

		cap = &mcx_eth_cap_map[i];
		if (cap->cap_media == 0)
			continue;

		ifmedia_add(&sc->sc_media, IFM_ETHER | cap->cap_media, 0, NULL);
	}

	proto_cap = betoh32(ptys.rp_ext_eth_proto_cap);
	for (i = 0; i < nitems(mcx_ext_eth_cap_map); i++) {
		const struct mcx_eth_proto_capability *cap;
		if (!ISSET(proto_cap, 1 << i))
			continue;

		cap = &mcx_ext_eth_cap_map[i];
		if (cap->cap_media == 0)
			continue;

		ifmedia_add(&sc->sc_media, IFM_ETHER | cap->cap_media, 0, NULL);
	}
}

static void
mcx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
{
	struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
	struct mcx_reg_ptys ptys;
	int i;
	uint32_t proto_oper;
	uint32_t ext_proto_oper;
	uint64_t media_oper;

	memset(&ptys, 0, sizeof(ptys));
	ptys.rp_local_port = 1;
	ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;

	rw_enter_write(&sc->sc_cmdq_ioctl_lk);
	if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys,
	    sizeof(ptys), MCX_CMDQ_SLOT_IOCTL) != 0) {
		printf("%s: unable to read port type/speed\n", DEVNAME(sc));
		goto out;
	}

	proto_oper = betoh32(ptys.rp_eth_proto_oper);
	ext_proto_oper = betoh32(ptys.rp_ext_eth_proto_oper);

	media_oper = 0;

	for (i = 0; i < nitems(mcx_eth_cap_map); i++) {
		const struct mcx_eth_proto_capability *cap;
		if (!ISSET(proto_oper, 1 << i))
			continue;

		cap = &mcx_eth_cap_map[i];

		if (cap->cap_media != 0)
			media_oper = cap->cap_media;
	}

	if (media_oper == 0) {
		for (i = 0; i < nitems(mcx_ext_eth_cap_map); i++) {
			const struct mcx_eth_proto_capability *cap;
			if (!ISSET(ext_proto_oper, 1 << i))
				continue;

			cap = &mcx_ext_eth_cap_map[i];

			if (cap->cap_media != 0)
				media_oper = cap->cap_media;
		}
	}

	ifmr->ifm_status = IFM_AVALID;
	if ((proto_oper | ext_proto_oper) != 0) {
		ifmr->ifm_status |= IFM_ACTIVE;
		ifmr->ifm_active = IFM_ETHER | IFM_AUTO | media_oper;
		/* txpause, rxpause, duplex? */
	}
 out:
	rw_exit_write(&sc->sc_cmdq_ioctl_lk);
}

static int
mcx_media_change(struct ifnet *ifp)
{
	struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
	struct mcx_reg_ptys ptys;
	struct mcx_reg_paos paos;
	uint32_t media;
	uint32_t ext_media;
	int i, error;

	if (IFM_TYPE(sc->sc_media.ifm_media) != IFM_ETHER)
		return EINVAL;

	error = 0;
	rw_enter_write(&sc->sc_cmdq_ioctl_lk);

	if (IFM_SUBTYPE(sc->sc_media.ifm_media) == IFM_AUTO) {
		/* read ptys to get supported media */
		memset(&ptys, 0, sizeof(ptys));
		ptys.rp_local_port = 1;
		ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
		if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ,
		    &ptys, sizeof(ptys), MCX_CMDQ_SLOT_IOCTL) != 0) {
			printf("%s: unable to read port type/speed\n",
			    DEVNAME(sc));
			error = EIO;
			goto out;
		}

		media = betoh32(ptys.rp_eth_proto_cap);
		ext_media = betoh32(ptys.rp_ext_eth_proto_cap);
	} else {
		/* map media type */
		media = 0;
		for (i = 0; i < nitems(mcx_eth_cap_map); i++) {
			const struct  mcx_eth_proto_capability *cap;

			cap = &mcx_eth_cap_map[i];
			if (cap->cap_media ==
			    IFM_SUBTYPE(sc->sc_media.ifm_media)) {
				media = (1 << i);
				break;
			}
		}

		ext_media = 0;
		for (i = 0; i < nitems(mcx_ext_eth_cap_map); i++) {
			const struct  mcx_eth_proto_capability *cap;

			cap = &mcx_ext_eth_cap_map[i];
			if (cap->cap_media ==
			    IFM_SUBTYPE(sc->sc_media.ifm_media)) {
				ext_media = (1 << i);
				break;
			}
		}
	}

	/* disable the port */
	memset(&paos, 0, sizeof(paos));
	paos.rp_local_port = 1;
	paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_DOWN;
	paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN;
	if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos,
	    sizeof(paos), MCX_CMDQ_SLOT_IOCTL) != 0) {
		printf("%s: unable to set port state to down\n", DEVNAME(sc));
		error = EIO;
		goto out;
	}

	memset(&ptys, 0, sizeof(ptys));
	ptys.rp_local_port = 1;
	ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
	ptys.rp_eth_proto_admin = htobe32(media);
	ptys.rp_ext_eth_proto_admin = htobe32(ext_media);
	if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_WRITE, &ptys,
	    sizeof(ptys), MCX_CMDQ_SLOT_IOCTL) != 0) {
		printf("%s: unable to set port media type/speed\n",
		    DEVNAME(sc));
		error = EIO;
		/* continue on */
	}

	/* re-enable the port to start negotiation */
	memset(&paos, 0, sizeof(paos));
	paos.rp_local_port = 1;
	paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_UP;
	paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN;
	if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos,
	    sizeof(paos), MCX_CMDQ_SLOT_IOCTL) != 0) {
		printf("%s: unable to set port state to up\n", DEVNAME(sc));
		error = EIO;
	}

 out:
	rw_exit_write(&sc->sc_cmdq_ioctl_lk);
	return error;
}

static void
mcx_port_change(void *xsc)
{
	struct mcx_softc *sc = xsc;
	struct ifnet *ifp = &sc->sc_ac.ac_if;
	struct mcx_reg_ptys ptys = {
		.rp_local_port = 1,
		.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH,
	};
	int link_state = LINK_STATE_DOWN;
	int slot;

	if (cold) {
		slot = MCX_CMDQ_SLOT_POLL;
	} else
		slot = MCX_CMDQ_SLOT_LINK;

	if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys,
	    sizeof(ptys), slot) == 0) {
		uint32_t proto_oper = betoh32(ptys.rp_eth_proto_oper);
		uint32_t ext_proto_oper = betoh32(ptys.rp_ext_eth_proto_oper);
		uint64_t baudrate = 0;
		unsigned int i;

		if ((proto_oper | ext_proto_oper) != 0)
			link_state = LINK_STATE_FULL_DUPLEX;

		for (i = 0; i < nitems(mcx_eth_cap_map); i++) {
			const struct mcx_eth_proto_capability *cap;
			if (!ISSET(proto_oper, 1 << i))
				continue;

			cap = &mcx_eth_cap_map[i];
			if (cap->cap_baudrate == 0)
				continue;

			baudrate = cap->cap_baudrate;
			break;
		}

		if (baudrate == 0) {
			for (i = 0; i < nitems(mcx_ext_eth_cap_map); i++) {
				const struct mcx_eth_proto_capability *cap;
				if (!ISSET(ext_proto_oper, 1 << i))
					continue;

				cap = &mcx_ext_eth_cap_map[i];
				if (cap->cap_baudrate == 0)
					continue;

				baudrate = cap->cap_baudrate;
				break;
			}
		}

		ifp->if_baudrate = baudrate;
	}

	if (link_state != ifp->if_link_state) {
		ifp->if_link_state = link_state;
		if_link_state_change(ifp);
	}
}

static inline uint32_t
mcx_rd(struct mcx_softc *sc, bus_size_t r)
{
	uint32_t word;

	word = bus_space_read_raw_4(sc->sc_memt, sc->sc_memh, r);

	return (betoh32(word));
}

static inline void
mcx_wr(struct mcx_softc *sc, bus_size_t r, uint32_t v)
{
	bus_space_write_raw_4(sc->sc_memt, sc->sc_memh, r, htobe32(v));
}

static inline void
mcx_bar(struct mcx_softc *sc, bus_size_t r, bus_size_t l, int f)
{
	bus_space_barrier(sc->sc_memt, sc->sc_memh, r, l, f);
}

static uint64_t
mcx_timer(struct mcx_softc *sc)
{
	uint32_t hi, lo, ni;

	hi = mcx_rd(sc, MCX_INTERNAL_TIMER_H);
	for (;;) {
		lo = mcx_rd(sc, MCX_INTERNAL_TIMER_L);
		mcx_bar(sc, MCX_INTERNAL_TIMER_L, 8, BUS_SPACE_BARRIER_READ);
		ni = mcx_rd(sc, MCX_INTERNAL_TIMER_H);

		if (ni == hi)
			break;

		hi = ni;
	}

	return (((uint64_t)hi << 32) | (uint64_t)lo);
}

static int
mcx_dmamem_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm,
    bus_size_t size, u_int align)
{
	mxm->mxm_size = size;

	if (bus_dmamap_create(sc->sc_dmat, mxm->mxm_size, 1,
	    mxm->mxm_size, 0,
	    BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT,
	    &mxm->mxm_map) != 0)
		return (1);
	if (bus_dmamem_alloc(sc->sc_dmat, mxm->mxm_size,
	    align, 0, &mxm->mxm_seg, 1, &mxm->mxm_nsegs,
	    BUS_DMA_WAITOK | BUS_DMA_ZERO) != 0)
		goto destroy;
	if (bus_dmamem_map(sc->sc_dmat, &mxm->mxm_seg, mxm->mxm_nsegs,
	    mxm->mxm_size, &mxm->mxm_kva, BUS_DMA_WAITOK) != 0)
		goto free;
	if (bus_dmamap_load(sc->sc_dmat, mxm->mxm_map, mxm->mxm_kva,
	    mxm->mxm_size, NULL, BUS_DMA_WAITOK) != 0)
		goto unmap;

	return (0);
unmap:
	bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size);
free:
	bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1);
destroy:
	bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map);
	return (1);
}

static void
mcx_dmamem_zero(struct mcx_dmamem *mxm)
{
	memset(MCX_DMA_KVA(mxm), 0, MCX_DMA_LEN(mxm));
}

static void
mcx_dmamem_free(struct mcx_softc *sc, struct mcx_dmamem *mxm)
{
	bus_dmamap_unload(sc->sc_dmat, mxm->mxm_map);
	bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size);
	bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1);
	bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map);
}

static int
mcx_hwmem_alloc(struct mcx_softc *sc, struct mcx_hwmem *mhm, unsigned int pages)
{
	bus_dma_segment_t *segs;
	bus_size_t len = pages * MCX_PAGE_SIZE;
	size_t seglen;

	segs = mallocarray(sizeof(*segs), pages, M_DEVBUF, M_WAITOK|M_CANFAIL);
	if (segs == NULL)
		return (-1);

	seglen = sizeof(*segs) * pages;

	if (bus_dmamem_alloc(sc->sc_dmat, len, MCX_PAGE_SIZE, 0,
	    segs, pages, &mhm->mhm_seg_count, BUS_DMA_NOWAIT) != 0)
		goto free_segs;

	if (mhm->mhm_seg_count < pages) {
		size_t nseglen;

		mhm->mhm_segs = mallocarray(sizeof(*mhm->mhm_segs),
		    mhm->mhm_seg_count, M_DEVBUF, M_WAITOK|M_CANFAIL);
		if (mhm->mhm_segs == NULL)
			goto free_dmamem;

		nseglen = sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count;

		memcpy(mhm->mhm_segs, segs, nseglen);

		free(segs, M_DEVBUF, seglen);

		segs = mhm->mhm_segs;
		seglen = nseglen;
	} else
		mhm->mhm_segs = segs;

	if (bus_dmamap_create(sc->sc_dmat, len, pages, MCX_PAGE_SIZE,
	    MCX_PAGE_SIZE, BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW /*|BUS_DMA_64BIT*/,
	    &mhm->mhm_map) != 0)
		goto free_dmamem;

	if (bus_dmamap_load_raw(sc->sc_dmat, mhm->mhm_map,
	    mhm->mhm_segs, mhm->mhm_seg_count, len, BUS_DMA_NOWAIT) != 0)
		goto destroy;

	bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map,
	    0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_PRERW);

	mhm->mhm_npages = pages;

	return (0);

destroy:
	bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map);
free_dmamem:
	bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count);
free_segs:
	free(segs, M_DEVBUF, seglen);
	mhm->mhm_segs = NULL;

	return (-1);
}

static void
mcx_hwmem_free(struct mcx_softc *sc, struct mcx_hwmem *mhm)
{
	if (mhm->mhm_npages == 0)
		return;

	bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map,
	    0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_POSTRW);

	bus_dmamap_unload(sc->sc_dmat, mhm->mhm_map);
	bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map);
	bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count);
	free(mhm->mhm_segs, M_DEVBUF,
	    sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count);

	mhm->mhm_npages = 0;
}

#if NKSTAT > 0
struct mcx_ppcnt {
	char			 name[KSTAT_KV_NAMELEN];
	enum kstat_kv_unit	 unit;
};

static const struct mcx_ppcnt mcx_ppcnt_ieee8023_tpl[] = {
	{ "Good Tx",		KSTAT_KV_U_PACKETS, },
	{ "Good Rx",		KSTAT_KV_U_PACKETS, },
	{ "FCS errs",		KSTAT_KV_U_PACKETS, },
	{ "Alignment Errs",	KSTAT_KV_U_PACKETS, },
	{ "Good Tx",		KSTAT_KV_U_BYTES, },
	{ "Good Rx",		KSTAT_KV_U_BYTES, },
	{ "Multicast Tx",	KSTAT_KV_U_PACKETS, },
	{ "Broadcast Tx",	KSTAT_KV_U_PACKETS, },
	{ "Multicast Rx",	KSTAT_KV_U_PACKETS, },
	{ "Broadcast Rx",	KSTAT_KV_U_PACKETS, },
	{ "In Range Len",	KSTAT_KV_U_PACKETS, },
	{ "Out Of Range Len",	KSTAT_KV_U_PACKETS, },
	{ "Frame Too Long",	KSTAT_KV_U_PACKETS, },
	{ "Symbol Errs",	KSTAT_KV_U_PACKETS, },
	{ "MAC Ctrl Tx",	KSTAT_KV_U_PACKETS, },
	{ "MAC Ctrl Rx",	KSTAT_KV_U_PACKETS, },
	{ "MAC Ctrl Unsup",	KSTAT_KV_U_PACKETS, },
	{ "Pause Rx",		KSTAT_KV_U_PACKETS, },
	{ "Pause Tx",		KSTAT_KV_U_PACKETS, },
};
CTASSERT(nitems(mcx_ppcnt_ieee8023_tpl) == mcx_ppcnt_ieee8023_count);

static const struct mcx_ppcnt mcx_ppcnt_rfc2863_tpl[] = {
	{ "Rx Bytes",		KSTAT_KV_U_BYTES, },
	{ "Rx Unicast",		KSTAT_KV_U_PACKETS, },
	{ "Rx Discards",	KSTAT_KV_U_PACKETS, },
	{ "Rx Errors",		KSTAT_KV_U_PACKETS, },
	{ "Rx Unknown Proto",	KSTAT_KV_U_PACKETS, },
	{ "Tx Bytes",		KSTAT_KV_U_BYTES, },
	{ "Tx Unicast",		KSTAT_KV_U_PACKETS, },
	{ "Tx Discards",	KSTAT_KV_U_PACKETS, },
	{ "Tx Errors",		KSTAT_KV_U_PACKETS, },
	{ "Rx Multicast",	KSTAT_KV_U_PACKETS, },
	{ "Rx Broadcast",	KSTAT_KV_U_PACKETS, },
	{ "Tx Multicast",	KSTAT_KV_U_PACKETS, },
	{ "Tx Broadcast",	KSTAT_KV_U_PACKETS, },
};
CTASSERT(nitems(mcx_ppcnt_rfc2863_tpl) == mcx_ppcnt_rfc2863_count);

static const struct mcx_ppcnt mcx_ppcnt_rfc2819_tpl[] = {
	{ "Drop Events",	KSTAT_KV_U_PACKETS, },
	{ "Octets",		KSTAT_KV_U_BYTES, },
	{ "Packets",		KSTAT_KV_U_PACKETS, },
	{ "Broadcasts",		KSTAT_KV_U_PACKETS, },
	{ "Multicasts",		KSTAT_KV_U_PACKETS, },
	{ "CRC Align Errs",	KSTAT_KV_U_PACKETS, },
	{ "Undersize",		KSTAT_KV_U_PACKETS, },
	{ "Oversize",		KSTAT_KV_U_PACKETS, },
	{ "Fragments",		KSTAT_KV_U_PACKETS, },
	{ "Jabbers",		KSTAT_KV_U_PACKETS, },
	{ "Collisions",		KSTAT_KV_U_NONE, },
	{ "64B",		KSTAT_KV_U_PACKETS, },
	{ "65-127B",		KSTAT_KV_U_PACKETS, },
	{ "128-255B",		KSTAT_KV_U_PACKETS, },
	{ "256-511B",		KSTAT_KV_U_PACKETS, },
	{ "512-1023B",		KSTAT_KV_U_PACKETS, },
	{ "1024-1518B",		KSTAT_KV_U_PACKETS, },
	{ "1519-2047B",		KSTAT_KV_U_PACKETS, },
	{ "2048-4095B",		KSTAT_KV_U_PACKETS, },
	{ "4096-8191B",		KSTAT_KV_U_PACKETS, },
	{ "8192-10239B",	KSTAT_KV_U_PACKETS, },
};
CTASSERT(nitems(mcx_ppcnt_rfc2819_tpl) == mcx_ppcnt_rfc2819_count);

static const struct mcx_ppcnt mcx_ppcnt_rfc3635_tpl[] = {
	{ "Alignment Errs",	KSTAT_KV_U_PACKETS, },
	{ "FCS Errs",		KSTAT_KV_U_PACKETS, },
	{ "Single Colls",	KSTAT_KV_U_PACKETS, },
	{ "Multiple Colls",	KSTAT_KV_U_PACKETS, },
	{ "SQE Test Errs",	KSTAT_KV_U_NONE, },
	{ "Deferred Tx",	KSTAT_KV_U_PACKETS, },
	{ "Late Colls",		KSTAT_KV_U_NONE, },
	{ "Exess Colls",	KSTAT_KV_U_NONE, },
	{ "Int MAC Tx Errs",	KSTAT_KV_U_PACKETS, },
	{ "CSM Sense Errs",	KSTAT_KV_U_NONE, },
	{ "Too Long",		KSTAT_KV_U_PACKETS, },
	{ "Int MAC Rx Errs",	KSTAT_KV_U_PACKETS, },
	{ "Symbol Errs",	KSTAT_KV_U_NONE, },
	{ "Unknown Control",	KSTAT_KV_U_PACKETS, },
	{ "Pause Rx",		KSTAT_KV_U_PACKETS, },
	{ "Pause Tx",		KSTAT_KV_U_PACKETS, },
};
CTASSERT(nitems(mcx_ppcnt_rfc3635_tpl) == mcx_ppcnt_rfc3635_count);

struct mcx_kstat_ppcnt {
	const char		*ksp_name;
	const struct mcx_ppcnt	*ksp_tpl;
	unsigned int		 ksp_n;
	uint8_t			 ksp_grp;
};

static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_ieee8023 = {
	.ksp_name =		"ieee802.3",
	.ksp_tpl =		mcx_ppcnt_ieee8023_tpl,
	.ksp_n =		nitems(mcx_ppcnt_ieee8023_tpl),
	.ksp_grp =		MCX_REG_PPCNT_GRP_IEEE8023,
};

static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2863 = {
	.ksp_name =		"rfc2863",
	.ksp_tpl =		mcx_ppcnt_rfc2863_tpl,
	.ksp_n =		nitems(mcx_ppcnt_rfc2863_tpl),
	.ksp_grp =		MCX_REG_PPCNT_GRP_RFC2863,
};

static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2819 = {
	.ksp_name =		"rfc2819",
	.ksp_tpl =		mcx_ppcnt_rfc2819_tpl,
	.ksp_n =		nitems(mcx_ppcnt_rfc2819_tpl),
	.ksp_grp =		MCX_REG_PPCNT_GRP_RFC2819,
};

static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc3635 = {
	.ksp_name =		"rfc3635",
	.ksp_tpl =		mcx_ppcnt_rfc3635_tpl,
	.ksp_n =		nitems(mcx_ppcnt_rfc3635_tpl),
	.ksp_grp =		MCX_REG_PPCNT_GRP_RFC3635,
};

static int	mcx_kstat_ppcnt_read(struct kstat *);

static void	mcx_kstat_attach_tmps(struct mcx_softc *sc);
static void	mcx_kstat_attach_queues(struct mcx_softc *sc);

static struct kstat *
mcx_kstat_attach_ppcnt(struct mcx_softc *sc,
    const struct mcx_kstat_ppcnt *ksp)
{
	struct kstat *ks;
	struct kstat_kv *kvs;
	unsigned int i;

	ks = kstat_create(DEVNAME(sc), 0, ksp->ksp_name, 0, KSTAT_T_KV, 0);
	if (ks == NULL)
		return (NULL);

	kvs = mallocarray(ksp->ksp_n, sizeof(*kvs),
	    M_DEVBUF, M_WAITOK);

	for (i = 0; i < ksp->ksp_n; i++) {
		const struct mcx_ppcnt *tpl = &ksp->ksp_tpl[i];

		kstat_kv_unit_init(&kvs[i], tpl->name,
		    KSTAT_KV_T_COUNTER64, tpl->unit);
	}

	ks->ks_softc = sc;
	ks->ks_ptr = (void *)ksp;
	ks->ks_data = kvs;
	ks->ks_datalen = ksp->ksp_n * sizeof(*kvs);
	ks->ks_read = mcx_kstat_ppcnt_read;
	kstat_set_wlock(ks, &sc->sc_cmdq_kstat_lk);

	kstat_install(ks);

	return (ks);
}

static void
mcx_kstat_attach(struct mcx_softc *sc)
{
	sc->sc_kstat_ieee8023 = mcx_kstat_attach_ppcnt(sc,
	    &mcx_kstat_ppcnt_ieee8023);
	sc->sc_kstat_rfc2863 = mcx_kstat_attach_ppcnt(sc,
	    &mcx_kstat_ppcnt_rfc2863);
	sc->sc_kstat_rfc2819 = mcx_kstat_attach_ppcnt(sc,
	    &mcx_kstat_ppcnt_rfc2819);
	sc->sc_kstat_rfc3635 = mcx_kstat_attach_ppcnt(sc,
	    &mcx_kstat_ppcnt_rfc3635);

	mcx_kstat_attach_tmps(sc);
	mcx_kstat_attach_queues(sc);
}

static int
mcx_kstat_ppcnt_read(struct kstat *ks)
{
	struct mcx_softc *sc = ks->ks_softc;
	struct mcx_kstat_ppcnt *ksp = ks->ks_ptr;
	struct mcx_reg_ppcnt ppcnt = {
		.ppcnt_grp = ksp->ksp_grp,
		.ppcnt_local_port = 1,
	};
	struct kstat_kv *kvs = ks->ks_data;
	uint64_t *vs = (uint64_t *)&ppcnt.ppcnt_counter_set;
	unsigned int i;
	int rv;

	rv = mcx_access_hca_reg(sc, MCX_REG_PPCNT, MCX_REG_OP_READ,
	    &ppcnt, sizeof(ppcnt), MCX_CMDQ_SLOT_KSTAT);
	if (rv != 0)
		return (EIO);

	nanouptime(&ks->ks_updated);

	for (i = 0; i < ksp->ksp_n; i++)
		kstat_kv_u64(&kvs[i]) = bemtoh64(&vs[i]);

	return (0);
}

struct mcx_kstat_mtmp {
	struct kstat_kv		ktmp_name;
	struct kstat_kv		ktmp_temperature;
	struct kstat_kv		ktmp_threshold_lo;
	struct kstat_kv		ktmp_threshold_hi;
};

static const struct mcx_kstat_mtmp mcx_kstat_mtmp_tpl = {
	KSTAT_KV_INITIALIZER("name",		KSTAT_KV_T_ISTR),
	KSTAT_KV_INITIALIZER("temperature",	KSTAT_KV_T_TEMP),
	KSTAT_KV_INITIALIZER("lo threshold",	KSTAT_KV_T_TEMP),
	KSTAT_KV_INITIALIZER("hi threshold",	KSTAT_KV_T_TEMP),
};

static const struct timeval mcx_kstat_mtmp_rate = { 1, 0 };

static int mcx_kstat_mtmp_read(struct kstat *);

static void
mcx_kstat_attach_tmps(struct mcx_softc *sc)
{
	struct kstat *ks;
	struct mcx_reg_mcam mcam;
	struct mcx_reg_mtcap mtcap;
	struct mcx_kstat_mtmp *ktmp;
	uint64_t map;
	unsigned int i, n;

	memset(&mtcap, 0, sizeof(mtcap));
	memset(&mcam, 0, sizeof(mcam));

	if (sc->sc_mcam_reg == 0) {
		/* no management capabilities */
		return;
	}

	if (mcx_access_hca_reg(sc, MCX_REG_MCAM, MCX_REG_OP_READ,
	    &mcam, sizeof(mcam), MCX_CMDQ_SLOT_POLL) != 0) {
		/* unable to check management capabilities? */
		return;
	}

	if (MCX_BITFIELD_BIT(mcam.mcam_feature_cap_mask,
	    MCX_MCAM_FEATURE_CAP_SENSOR_MAP) == 0) {
		/* no sensor map */
		return;
	}

	if (mcx_access_hca_reg(sc, MCX_REG_MTCAP, MCX_REG_OP_READ,
	    &mtcap, sizeof(mtcap), MCX_CMDQ_SLOT_POLL) != 0) {
		/* unable to find temperature sensors */
		return;
	}

	sc->sc_kstat_mtmp_count = mtcap.mtcap_sensor_count;
	sc->sc_kstat_mtmp = mallocarray(sc->sc_kstat_mtmp_count,
	    sizeof(*sc->sc_kstat_mtmp), M_DEVBUF, M_WAITOK);

	n = 0;
	map = bemtoh64(&mtcap.mtcap_sensor_map);
	for (i = 0; i < sizeof(map) * NBBY; i++) {
		if (!ISSET(map, (1ULL << i)))
			continue;

		ks = kstat_create(DEVNAME(sc), 0, "temperature", i,
		    KSTAT_T_KV, 0);
		if (ks == NULL) {
			/* unable to attach temperature sensor %u, i */
			continue;
		}

		ktmp = malloc(sizeof(*ktmp), M_DEVBUF, M_WAITOK|M_ZERO);
		*ktmp = mcx_kstat_mtmp_tpl;

		ks->ks_data = ktmp;
		ks->ks_datalen = sizeof(*ktmp);
		TIMEVAL_TO_TIMESPEC(&mcx_kstat_mtmp_rate, &ks->ks_interval);
		ks->ks_read = mcx_kstat_mtmp_read;
		kstat_set_wlock(ks, &sc->sc_cmdq_kstat_lk);

		ks->ks_softc = sc;
		kstat_install(ks);

		sc->sc_kstat_mtmp[n++] = ks;
		if (n >= sc->sc_kstat_mtmp_count)
			break;
	}
}

static uint64_t
mcx_tmp_to_uK(uint16_t *t)
{
	int64_t mt = (int16_t)bemtoh16(t); /* 0.125 C units */
	mt *= 1000000 / 8; /* convert to uC */
	mt += 273150000; /* convert to uK */

	return (mt);
}

static int
mcx_kstat_mtmp_read(struct kstat *ks)
{
	struct mcx_softc *sc = ks->ks_softc;
	struct mcx_kstat_mtmp *ktmp = ks->ks_data;
	struct mcx_reg_mtmp mtmp;
	int rv;
	struct timeval updated;

	TIMESPEC_TO_TIMEVAL(&updated, &ks->ks_updated);

	if (!ratecheck(&updated, &mcx_kstat_mtmp_rate))
		return (0);

	memset(&mtmp, 0, sizeof(mtmp));
	htobem16(&mtmp.mtmp_sensor_index, ks->ks_unit);

	rv = mcx_access_hca_reg(sc, MCX_REG_MTMP, MCX_REG_OP_READ,
	    &mtmp, sizeof(mtmp), MCX_CMDQ_SLOT_KSTAT);
	if (rv != 0)
		return (EIO);

	memset(kstat_kv_istr(&ktmp->ktmp_name), 0,
	    sizeof(kstat_kv_istr(&ktmp->ktmp_name)));
	memcpy(kstat_kv_istr(&ktmp->ktmp_name),
	    mtmp.mtmp_sensor_name, sizeof(mtmp.mtmp_sensor_name));
	kstat_kv_temp(&ktmp->ktmp_temperature) =
	    mcx_tmp_to_uK(&mtmp.mtmp_temperature);
	kstat_kv_temp(&ktmp->ktmp_threshold_lo) =
	    mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_lo);
	kstat_kv_temp(&ktmp->ktmp_threshold_hi) =
	    mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_hi);

	TIMEVAL_TO_TIMESPEC(&updated, &ks->ks_updated);

	return (0);
}

struct mcx_queuestat {
	char			 name[KSTAT_KV_NAMELEN];
	enum kstat_kv_type	 type;
};

static const struct mcx_queuestat mcx_queue_kstat_tpl[] = {
	{ "RQ SW prod",		KSTAT_KV_T_COUNTER64 },
	{ "RQ HW prod",		KSTAT_KV_T_COUNTER64 },
	{ "RQ HW cons",		KSTAT_KV_T_COUNTER64 },
	{ "RQ HW state",	KSTAT_KV_T_ISTR },

	{ "SQ SW prod",		KSTAT_KV_T_COUNTER64 },
	{ "SQ SW cons",		KSTAT_KV_T_COUNTER64 },
	{ "SQ HW prod",		KSTAT_KV_T_COUNTER64 },
	{ "SQ HW cons",		KSTAT_KV_T_COUNTER64 },
	{ "SQ HW state",	KSTAT_KV_T_ISTR },

	{ "CQ SW cons",		KSTAT_KV_T_COUNTER64 },
	{ "CQ HW prod",		KSTAT_KV_T_COUNTER64 },
	{ "CQ HW cons",		KSTAT_KV_T_COUNTER64 },
	{ "CQ HW notify",	KSTAT_KV_T_COUNTER64 },
	{ "CQ HW solicit",	KSTAT_KV_T_COUNTER64 },
	{ "CQ HW status",	KSTAT_KV_T_ISTR },
	{ "CQ HW state",	KSTAT_KV_T_ISTR },

	{ "EQ SW cons",		KSTAT_KV_T_COUNTER64 },
	{ "EQ HW prod",		KSTAT_KV_T_COUNTER64 },
	{ "EQ HW cons",		KSTAT_KV_T_COUNTER64 },
	{ "EQ HW status",	KSTAT_KV_T_ISTR },
	{ "EQ HW state",	KSTAT_KV_T_ISTR },
};

static int	mcx_kstat_queue_read(struct kstat *);

static void
mcx_kstat_attach_queues(struct mcx_softc *sc)
{
	struct kstat *ks;
	struct kstat_kv *kvs;
	int q, i;

	for (q = 0; q < intrmap_count(sc->sc_intrmap); q++) {
		ks = kstat_create(DEVNAME(sc), 0, "mcx-queues", q,
		    KSTAT_T_KV, 0);
		if (ks == NULL) {
			/* unable to attach queue stats %u, q */
			continue;
		}

		kvs = mallocarray(nitems(mcx_queue_kstat_tpl),
		    sizeof(*kvs), M_DEVBUF, M_WAITOK);

		for (i = 0; i < nitems(mcx_queue_kstat_tpl); i++) {
			const struct mcx_queuestat *tpl =
			    &mcx_queue_kstat_tpl[i];

			kstat_kv_init(&kvs[i], tpl->name, tpl->type);
		}

		ks->ks_softc = &sc->sc_queues[q];
		ks->ks_data = kvs;
		ks->ks_datalen = nitems(mcx_queue_kstat_tpl) * sizeof(*kvs);
		ks->ks_read = mcx_kstat_queue_read;

		sc->sc_queues[q].q_kstat = ks;
		kstat_install(ks);
	}
}

static int
mcx_kstat_queue_read(struct kstat *ks)
{
	struct mcx_queues *q = ks->ks_softc;
	struct mcx_softc *sc = q->q_sc;
	struct kstat_kv *kvs = ks->ks_data;
	union {
		struct mcx_rq_ctx rq;
		struct mcx_sq_ctx sq;
		struct mcx_cq_ctx cq;
		struct mcx_eq_ctx eq;
	} u;
	const char *text;
	int error = 0;

	if (mcx_query_rq(sc, &q->q_rx, &u.rq) != 0) {
		error = EIO;
		goto out;
	}

	kstat_kv_u64(kvs++) = q->q_rx.rx_prod;
	kstat_kv_u64(kvs++) = bemtoh32(&u.rq.rq_wq.wq_sw_counter);
	kstat_kv_u64(kvs++) = bemtoh32(&u.rq.rq_wq.wq_hw_counter);
	switch ((bemtoh32(&u.rq.rq_flags) & MCX_RQ_CTX_STATE_MASK) >>
	    MCX_RQ_CTX_STATE_SHIFT) {
	case MCX_RQ_CTX_STATE_RST:
		text = "RST";
		break;
	case MCX_RQ_CTX_STATE_RDY:
		text = "RDY";
		break;
	case MCX_RQ_CTX_STATE_ERR:
		text = "ERR";
		break;
	default:
		text = "unknown";
		break;
	}
	strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
	kvs++;

	if (mcx_query_sq(sc, &q->q_tx, &u.sq) != 0) {
		error = EIO;
		goto out;
	}

	kstat_kv_u64(kvs++) = q->q_tx.tx_prod;
	kstat_kv_u64(kvs++) = q->q_tx.tx_cons;
	kstat_kv_u64(kvs++) = bemtoh32(&u.sq.sq_wq.wq_sw_counter);
	kstat_kv_u64(kvs++) = bemtoh32(&u.sq.sq_wq.wq_hw_counter);
	switch ((bemtoh32(&u.sq.sq_flags) & MCX_SQ_CTX_STATE_MASK) >>
	    MCX_SQ_CTX_STATE_SHIFT) {
	case MCX_SQ_CTX_STATE_RST:
		text = "RST";
		break;
	case MCX_SQ_CTX_STATE_RDY:
		text = "RDY";
		break;
	case MCX_SQ_CTX_STATE_ERR:
		text = "ERR";
		break;
	default:
		text = "unknown";
		break;
	}
	strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
	kvs++;

	if (mcx_query_cq(sc, &q->q_cq, &u.cq) != 0) {
		error = EIO;
		goto out;
	}

	kstat_kv_u64(kvs++) = q->q_cq.cq_cons;
	kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_producer_counter);
	kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_consumer_counter);
	kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_notified);
	kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_solicit);

	switch ((bemtoh32(&u.cq.cq_status) & MCX_CQ_CTX_STATUS_MASK) >>
	    MCX_CQ_CTX_STATUS_SHIFT) {
	case MCX_CQ_CTX_STATUS_OK:
		text = "OK";
		break;
	case MCX_CQ_CTX_STATUS_OVERFLOW:
		text = "overflow";
		break;
	case MCX_CQ_CTX_STATUS_WRITE_FAIL:
		text = "write fail";
		break;
	default:
		text = "unknown";
		break;
	}
	strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
	kvs++;

	switch ((bemtoh32(&u.cq.cq_status) & MCX_CQ_CTX_STATE_MASK) >>
	    MCX_CQ_CTX_STATE_SHIFT) {
	case MCX_CQ_CTX_STATE_SOLICITED:
		text = "solicited";
		break;
	case MCX_CQ_CTX_STATE_ARMED:
		text = "armed";
		break;
	case MCX_CQ_CTX_STATE_FIRED:
		text = "fired";
		break;
	default:
		text = "unknown";
		break;
	}
	strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
	kvs++;

	if (mcx_query_eq(sc, &q->q_eq, &u.eq) != 0) {
		error = EIO;
		goto out;
	}

	kstat_kv_u64(kvs++) = q->q_eq.eq_cons;
	kstat_kv_u64(kvs++) = bemtoh32(&u.eq.eq_producer_counter);
	kstat_kv_u64(kvs++) = bemtoh32(&u.eq.eq_consumer_counter);

	switch ((bemtoh32(&u.eq.eq_status) & MCX_EQ_CTX_STATUS_MASK) >>
	    MCX_EQ_CTX_STATUS_SHIFT) {
	case MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE:
		text = "write fail";
		break;
	case MCX_EQ_CTX_STATUS_OK:
		text = "OK";
		break;
	default:
		text = "unknown";
		break;
	}
	strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
	kvs++;

	switch ((bemtoh32(&u.eq.eq_status) & MCX_EQ_CTX_STATE_MASK) >>
	    MCX_EQ_CTX_STATE_SHIFT) {
	case MCX_EQ_CTX_STATE_ARMED:
		text = "armed";
		break;
	case MCX_EQ_CTX_STATE_FIRED:
		text = "fired";
		break;
	default:
		text = "unknown";
		break;
	}
	strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
	kvs++;

	nanouptime(&ks->ks_updated);
out:
	return (error);
}

#endif /* NKSTAT > 0 */

static unsigned int
mcx_timecounter_read(struct timecounter *tc)
{
	struct mcx_softc *sc = tc->tc_priv;

	return (mcx_rd(sc, MCX_INTERNAL_TIMER_L));
}

static void
mcx_timecounter_attach(struct mcx_softc *sc)
{
	struct timecounter *tc = &sc->sc_timecounter;

	tc->tc_get_timecount = mcx_timecounter_read;
	tc->tc_counter_mask = ~0U;
	tc->tc_frequency = sc->sc_khz * 1000;
	tc->tc_name = sc->sc_dev.dv_xname;
	tc->tc_quality = -100;
	tc->tc_priv = sc;

	tc_init(tc);
}