path: root/sys/dev/pci/if_ice.c

/*	$OpenBSD: if_ice.c,v 1.9 2024/11/15 15:43:49 stsp Exp $	*/

/*  Copyright (c) 2024, Intel Corporation
 *  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions are met:
 *
 *   1. Redistributions of source code must retain the above copyright notice,
 *      this list of conditions and the following disclaimer.
 *
 *   2. Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in the
 *      documentation and/or other materials provided with the distribution.
 *
 *   3. Neither the name of the Intel Corporation nor the names of its
 *      contributors may be used to endorse or promote products derived from
 *      this software without specific prior written permission.
 *
 *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 *  POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * Ported from FreeBSD ice(4) by Stefan Sperling in 2024.
 *
 * Copyright (c) 2024 Stefan Sperling <stsp@openbsd.org>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include "bpfilter.h"

#include <sys/param.h>
#include <sys/atomic.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/rwlock.h>
#include <sys/mutex.h>
#include <sys/socket.h>
#include <sys/sockio.h>
#include <sys/systm.h>
#include <sys/endian.h>
#include <sys/intrmap.h>
#include <sys/kernel.h>

#include <sys/refcnt.h>
#include <sys/task.h>
#include <machine/bus.h>
#include <machine/intr.h>

#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <dev/pci/pcidevs.h>

#if NBPFILTER > 0
#include <net/bpf.h>
#endif
#include <net/if.h>
#include <net/if_media.h>

#include <netinet/in.h>
#include <netinet/if_ether.h>

#define STRUCT_HACK_VAR_LEN

/**
 * ice_struct_size - size of struct with C99 flexible array member
 * @ptr: pointer to structure
 * @field: flexible array member (last member of the structure)
 * @num: number of elements of that flexible array member
 */
#define ice_struct_size(ptr, field, num) \
	(sizeof(*(ptr)) + sizeof(*(ptr)->field) * (num))

#define FLEX_ARRAY_SIZE(_ptr, _mem, cnt) ((cnt) * sizeof(_ptr->_mem[0]))

#include "if_icereg.h"
#include "if_icevar.h"

/**
 * @var ice_driver_version
 * @brief driver version string
 *
 * Driver version information, used as part of the driver information
 * sent to the firmware at load.
 *
 * @var ice_major_version
 * @brief driver major version number
 *
 * @var ice_minor_version
 * @brief driver minor version number
 *
 * @var ice_patch_version
 * @brief driver patch version number
 *
 * @var ice_rc_version
 * @brief driver release candidate version number
 */
const char ice_driver_version[] = "1.39.13-k";
const uint8_t ice_major_version = 1;
const uint8_t ice_minor_version = 39;
const uint8_t ice_patch_version = 13;
const uint8_t ice_rc_version = 0;

typedef void *ice_match_t;

static const struct pci_matchid ice_devices[] = {
	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_E810_XXV_SFP },
#if 0 /* no hardware available for testing: */
	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_E810_XXV_QSFP },
#endif
};

int
ice_match(struct device *parent, ice_match_t match __unused, void *aux)
{
	struct pci_attach_args *pa = aux;
	return pci_matchbyid(pa, ice_devices, nitems(ice_devices));
}

#define ICE_DEBUG
#ifdef ICE_DEBUG
#define DPRINTF(x...)		do { if (ice_debug) printf(x); } while(0)
#define DNPRINTF(n,x...)	do { if (ice_debug & n) printf(x); } while(0)
#define ICE_DBG_TRACE		(1UL << 0) /* for function-trace only */
#define ICE_DBG_INIT		(1UL << 1)
#define ICE_DBG_RELEASE		(1UL << 2)
#define ICE_DBG_FW_LOG		(1UL << 3)
#define ICE_DBG_LINK		(1UL << 4)
#define ICE_DBG_PHY		(1UL << 5)
#define ICE_DBG_QCTX		(1UL << 6)
#define ICE_DBG_NVM		(1UL << 7)
#define ICE_DBG_LAN		(1UL << 8)
#define ICE_DBG_FLOW		(1UL << 9)
#define ICE_DBG_DCB		(1UL << 10)
#define ICE_DBG_DIAG		(1UL << 11)
#define ICE_DBG_FD		(1UL << 12)
#define ICE_DBG_SW		(1UL << 13)
#define ICE_DBG_SCHED		(1UL << 14)
#define ICE_DBG_RDMA		(1UL << 15)
#define ICE_DBG_PKG		(1UL << 16)
#define ICE_DBG_RES		(1UL << 17)
#define ICE_DBG_AQ_MSG		(1UL << 24)
#define ICE_DBG_AQ_DESC		(1UL << 25)
#define ICE_DBG_AQ_DESC_BUF	(1UL << 26)
#define ICE_DBG_AQ_CMD		(1UL << 27)
#define ICE_DBG_AQ		(ICE_DBG_AQ_MSG		| \
				 ICE_DBG_AQ_DESC	| \
				 ICE_DBG_AQ_DESC_BUF	| \
				 ICE_DBG_AQ_CMD)
#define ICE_DBG_PARSER		(1UL << 28)
#define ICE_DBG_USER		(1UL << 31)
uint32_t	ice_debug = 0xffffffff & ~(ICE_DBG_AQ);
#else
#define DPRINTF(x...)
#define DNPRINTF(n,x...)
#endif

#define ICE_READ(hw, reg)						\
	bus_space_read_4((hw)->hw_sc->sc_st, (hw)->hw_sc->sc_sh, (reg))

#define ICE_WRITE(hw, reg, val)						\
	bus_space_write_4((hw)->hw_sc->sc_st, (hw)->hw_sc->sc_sh, (reg), (val))

#define ice_flush(_hw) ICE_READ((_hw), GLGEN_STAT)

/* Data type manipulation macros. */
#define ICE_HI_DWORD(x)		((uint32_t)((((x) >> 16) >> 16) & 0xFFFFFFFF))
#define ICE_LO_DWORD(x)		((uint32_t)((x) & 0xFFFFFFFF))
#define ICE_HI_WORD(x)		((uint16_t)(((x) >> 16) & 0xFFFF))
#define ICE_LO_WORD(x)		((uint16_t)((x) & 0xFFFF))
#define ICE_HI_BYTE(x)		((uint8_t)(((x) >> 8) & 0xFF))
#define ICE_LO_BYTE(x)		((uint8_t)((x) & 0xFF))

uint16_t ice_lock_count;

/**
 * @enum feat_list
 * @brief driver feature enumeration
 *
 * Enumeration of possible device driver features that can be enabled or
 * disabled. Each possible value represents a different feature which can be
 * enabled or disabled.
 *
 * The driver stores a bitmap of the features that the device and OS are
 * capable of, as well as another bitmap indicating which features are
 * currently enabled for that device.
 */
enum feat_list {
	ICE_FEATURE_SRIOV,
	ICE_FEATURE_RSS,
	ICE_FEATURE_NETMAP,
	ICE_FEATURE_FDIR,
	ICE_FEATURE_MSI,
	ICE_FEATURE_MSIX,
	ICE_FEATURE_RDMA,
	ICE_FEATURE_SAFE_MODE,
	ICE_FEATURE_LENIENT_LINK_MODE,
	ICE_FEATURE_LINK_MGMT_VER_1,
	ICE_FEATURE_LINK_MGMT_VER_2,
	ICE_FEATURE_HEALTH_STATUS,
	ICE_FEATURE_FW_LOGGING,
	ICE_FEATURE_HAS_PBA,
	ICE_FEATURE_DCB,
	ICE_FEATURE_TX_BALANCE,
	ICE_FEATURE_DUAL_NAC,
	ICE_FEATURE_TEMP_SENSOR,
	/* Must be last entry */
	ICE_FEATURE_COUNT
};

struct ice_intr_vector {
	struct ice_softc	*iv_sc;
	struct ice_rx_queue	*iv_rxq;
	struct ice_tx_queue	*iv_txq;
	int			 iv_qid;
	void			*iv_ihc;
	char			 iv_name[16];
	pci_intr_handle_t	 ih;
};

#define ICE_MAX_VECTORS			8 /* XXX this is pretty arbitrary */

struct ice_softc {
	struct device sc_dev;
	struct arpcom		sc_ac;
	struct ifmedia		media;

	bus_space_tag_t sc_st;
	bus_space_handle_t sc_sh;
	bus_size_t sc_sz;
	bus_dma_tag_t sc_dmat;
	pci_product_id_t sc_pid;
	pci_chipset_tag_t sc_pct;
	pcitag_t sc_pcitag;

	pci_intr_handle_t sc_ih;
	void *sc_ihc;
	unsigned int sc_nmsix_max;
	unsigned int sc_nmsix;
	unsigned int sc_nqueues;
	struct intrmap *sc_intrmap;
	struct ice_intr_vector *sc_vectors;
	size_t sc_nvectors;

	struct task sc_admin_task;
	struct timeout sc_admin_timer;

	unsigned int sc_dead;

	enum ice_state state;
	struct ice_hw hw;

	struct ice_vsi pf_vsi;		/* Main PF VSI */

	/* Tri-state feature flags (capable/enabled) */
	ice_declare_bitmap(feat_cap, ICE_FEATURE_COUNT);
	ice_declare_bitmap(feat_en, ICE_FEATURE_COUNT);

	struct ice_resmgr os_imgr;

	/* isc_* fields inherited from FreeBSD iflib struct if_softc_ctx */
	int isc_vectors;
	int isc_nrxqsets;
	int isc_ntxqsets;
	int isc_msix_bar;
	int isc_tx_nsegments;
	int isc_ntxd[8];
	int isc_nrxd[8];
	uint32_t isc_txqsizes[8];
	uint32_t isc_rxqsizes[8];
	uint8_t isc_txd_size[8];
	uint8_t isc_rxd_size[8];
	int isc_tx_tso_segments_max;
	int isc_tx_tso_size_max;
	int isc_tx_tso_segsize_max;
	int isc_tx_csum_flags;
	int isc_capabilities;
	int isc_capenable;
	int isc_rss_table_size;
	int isc_rss_table_mask;
	int isc_nrxqsets_max;
	int isc_ntxqsets_max;
	uint16_t isc_rxd_buf_size[8]; /* set at init time by driver, 0
				         means use iflib-calculated size
				         based on isc_max_frame_size */
	uint16_t isc_max_frame_size; /* set at init time by driver */
	uint16_t isc_min_frame_size; /* set at init time by driver, only used if
					IFLIB_NEED_ETHER_PAD is set. */
	uint32_t isc_pause_frames;   /* set by driver for iflib_timer to detect */
	int isc_disable_msix;

	/* Tx/Rx queue managers */
	struct ice_resmgr tx_qmgr;
	struct ice_resmgr rx_qmgr;

	struct ice_vsi **all_vsi;	/* Array of VSI pointers */
	uint16_t num_available_vsi;	/* Size of VSI array */

	/* Interrupt allocation manager */
	struct ice_resmgr dev_imgr;
	uint16_t *pf_imap;
	int lan_vectors;

	/* NVM link override settings */
	struct ice_link_default_override_tlv ldo_tlv;

	bool link_up;

	int rebuild_ticks;

	int sw_intr[ICE_MAX_VECTORS];
};

/**
 * ice_driver_is_detaching - Check if the driver is detaching/unloading
 * @sc: device private softc
 *
 * Returns true if the driver is detaching, false otherwise.
 *
 * @remark on newer kernels, take advantage of iflib_in_detach in order to
 * report detachment correctly as early as possible.
 *
 * @remark this function is used by various code paths that want to avoid
 * running if the driver is about to be removed. This includes sysctls and
 * other driver access points. Note that it does not fully resolve
 * detach-based race conditions as it is possible for a thread to race with
 * iflib_in_detach.
 */
bool
ice_driver_is_detaching(struct ice_softc *sc)
{
	return (ice_test_state(&sc->state, ICE_STATE_DETACHING) || sc->sc_dead);
}

/*
 * ice_usec_delay - Delay for the specified number of microseconds
 * @time: microseconds to delay
 * @sleep: if true, sleep where possible
 *
 * If sleep is true, sleep so that another thread can execute.
 * Otherwise, use DELAY to spin the thread instead.
 */
void
ice_usec_delay(uint32_t time, bool sleep)
{
	if (sleep && !cold)
		tsleep_nsec(&sleep, 0, "icedly", USEC_TO_NSEC(time));
	else
		DELAY(time);
}

/*
 * ice_msec_delay - Delay for the specified number of milliseconds
 * @time: milliseconds to delay
 * @sleep: if true, sleep where possible
 *
 * If sleep is true, sleep so that another thread can execute.
 * Otherwise, use DELAY to spin the thread instead.
 */
void
ice_msec_delay(uint32_t time, bool sleep)
{
	if (sleep && !cold)
		tsleep_nsec(&sleep, 0, "icedly", MSEC_TO_NSEC(time));
	else
		DELAY(time * 1000);
}

/**
 * ice_aq_str - Convert an AdminQ error into a string
 * @aq_err: the AQ error code to convert
 *
 * Convert the AdminQ status into its string name, if known. Otherwise, format
 * the error as an integer.
 */
const char *
ice_aq_str(enum ice_aq_err aq_err)
{
	static char buf[ICE_STR_BUF_LEN];
	const char *str = NULL;

	switch (aq_err) {
	case ICE_AQ_RC_OK:
		str = "OK";
		break;
	case ICE_AQ_RC_EPERM:
		str = "AQ_RC_EPERM";
		break;
	case ICE_AQ_RC_ENOENT:
		str = "AQ_RC_ENOENT";
		break;
	case ICE_AQ_RC_ESRCH:
		str = "AQ_RC_ESRCH";
		break;
	case ICE_AQ_RC_EINTR:
		str = "AQ_RC_EINTR";
		break;
	case ICE_AQ_RC_EIO:
		str = "AQ_RC_EIO";
		break;
	case ICE_AQ_RC_ENXIO:
		str = "AQ_RC_ENXIO";
		break;
	case ICE_AQ_RC_E2BIG:
		str = "AQ_RC_E2BIG";
		break;
	case ICE_AQ_RC_EAGAIN:
		str = "AQ_RC_EAGAIN";
		break;
	case ICE_AQ_RC_ENOMEM:
		str = "AQ_RC_ENOMEM";
		break;
	case ICE_AQ_RC_EACCES:
		str = "AQ_RC_EACCES";
		break;
	case ICE_AQ_RC_EFAULT:
		str = "AQ_RC_EFAULT";
		break;
	case ICE_AQ_RC_EBUSY:
		str = "AQ_RC_EBUSY";
		break;
	case ICE_AQ_RC_EEXIST:
		str = "AQ_RC_EEXIST";
		break;
	case ICE_AQ_RC_EINVAL:
		str = "AQ_RC_EINVAL";
		break;
	case ICE_AQ_RC_ENOTTY:
		str = "AQ_RC_ENOTTY";
		break;
	case ICE_AQ_RC_ENOSPC:
		str = "AQ_RC_ENOSPC";
		break;
	case ICE_AQ_RC_ENOSYS:
		str = "AQ_RC_ENOSYS";
		break;
	case ICE_AQ_RC_ERANGE:
		str = "AQ_RC_ERANGE";
		break;
	case ICE_AQ_RC_EFLUSHED:
		str = "AQ_RC_EFLUSHED";
		break;
	case ICE_AQ_RC_BAD_ADDR:
		str = "AQ_RC_BAD_ADDR";
		break;
	case ICE_AQ_RC_EMODE:
		str = "AQ_RC_EMODE";
		break;
	case ICE_AQ_RC_EFBIG:
		str = "AQ_RC_EFBIG";
		break;
	case ICE_AQ_RC_ESBCOMP:
		str = "AQ_RC_ESBCOMP";
		break;
	case ICE_AQ_RC_ENOSEC:
		str = "AQ_RC_ENOSEC";
		break;
	case ICE_AQ_RC_EBADSIG:
		str = "AQ_RC_EBADSIG";
		break;
	case ICE_AQ_RC_ESVN:
		str = "AQ_RC_ESVN";
		break;
	case ICE_AQ_RC_EBADMAN:
		str = "AQ_RC_EBADMAN";
		break;
	case ICE_AQ_RC_EBADBUF:
		str = "AQ_RC_EBADBUF";
		break;
	case ICE_AQ_RC_EACCES_BMCU:
		str = "AQ_RC_EACCES_BMCU";
		break;
	}

	if (str)
		snprintf(buf, ICE_STR_BUF_LEN, "%s", str);
	else
		snprintf(buf, ICE_STR_BUF_LEN, "%d", aq_err);

	return buf;
}

/**
 * ice_status_str - convert status err code to a string
 * @status: the status error code to convert
 *
 * Convert the status code into its string name if known.
 *
 * Otherwise, use the scratch space to format the status code into a number.
 */
const char *
ice_status_str(enum ice_status status)
{
	static char buf[ICE_STR_BUF_LEN];
	const char *str = NULL;

	switch (status) {
	case ICE_SUCCESS:
		str = "OK";
		break;
	case ICE_ERR_PARAM:
		str = "ICE_ERR_PARAM";
		break;
	case ICE_ERR_NOT_IMPL:
		str = "ICE_ERR_NOT_IMPL";
		break;
	case ICE_ERR_NOT_READY:
		str = "ICE_ERR_NOT_READY";
		break;
	case ICE_ERR_NOT_SUPPORTED:
		str = "ICE_ERR_NOT_SUPPORTED";
		break;
	case ICE_ERR_BAD_PTR:
		str = "ICE_ERR_BAD_PTR";
		break;
	case ICE_ERR_INVAL_SIZE:
		str = "ICE_ERR_INVAL_SIZE";
		break;
	case ICE_ERR_DEVICE_NOT_SUPPORTED:
		str = "ICE_ERR_DEVICE_NOT_SUPPORTED";
		break;
	case ICE_ERR_RESET_FAILED:
		str = "ICE_ERR_RESET_FAILED";
		break;
	case ICE_ERR_FW_API_VER:
		str = "ICE_ERR_FW_API_VER";
		break;
	case ICE_ERR_NO_MEMORY:
		str = "ICE_ERR_NO_MEMORY";
		break;
	case ICE_ERR_CFG:
		str = "ICE_ERR_CFG";
		break;
	case ICE_ERR_OUT_OF_RANGE:
		str = "ICE_ERR_OUT_OF_RANGE";
		break;
	case ICE_ERR_ALREADY_EXISTS:
		str = "ICE_ERR_ALREADY_EXISTS";
		break;
	case ICE_ERR_NVM:
		str = "ICE_ERR_NVM";
		break;
	case ICE_ERR_NVM_CHECKSUM:
		str = "ICE_ERR_NVM_CHECKSUM";
		break;
	case ICE_ERR_BUF_TOO_SHORT:
		str = "ICE_ERR_BUF_TOO_SHORT";
		break;
	case ICE_ERR_NVM_BLANK_MODE:
		str = "ICE_ERR_NVM_BLANK_MODE";
		break;
	case ICE_ERR_IN_USE:
		str = "ICE_ERR_IN_USE";
		break;
	case ICE_ERR_MAX_LIMIT:
		str = "ICE_ERR_MAX_LIMIT";
		break;
	case ICE_ERR_RESET_ONGOING:
		str = "ICE_ERR_RESET_ONGOING";
		break;
	case ICE_ERR_HW_TABLE:
		str = "ICE_ERR_HW_TABLE";
		break;
	case ICE_ERR_FW_DDP_MISMATCH:
		str = "ICE_ERR_FW_DDP_MISMATCH";
		break;
	case ICE_ERR_DOES_NOT_EXIST:
		str = "ICE_ERR_DOES_NOT_EXIST";
		break;
	case ICE_ERR_AQ_ERROR:
		str = "ICE_ERR_AQ_ERROR";
		break;
	case ICE_ERR_AQ_TIMEOUT:
		str = "ICE_ERR_AQ_TIMEOUT";
		break;
	case ICE_ERR_AQ_FULL:
		str = "ICE_ERR_AQ_FULL";
		break;
	case ICE_ERR_AQ_NO_WORK:
		str = "ICE_ERR_AQ_NO_WORK";
		break;
	case ICE_ERR_AQ_EMPTY:
		str = "ICE_ERR_AQ_EMPTY";
		break;
	case ICE_ERR_AQ_FW_CRITICAL:
		str = "ICE_ERR_AQ_FW_CRITICAL";
		break;
	}

	if (str)
		snprintf(buf, ICE_STR_BUF_LEN, "%s", str);
	else
		snprintf(buf, ICE_STR_BUF_LEN, "%d", status);

	return buf;
}

/**
 * ice_mdd_tx_tclan_str - Convert MDD Tx TCLAN event to a string
 * @event: the MDD event number to convert
 *
 * Convert the Tx TCLAN event value from the GL_MDET_TX_TCLAN register into
 * a human readable string for logging of MDD events.
 */
const char *
ice_mdd_tx_tclan_str(uint8_t event)
{
	static char buf[ICE_STR_BUF_LEN];
	const char *str = NULL;

	switch (event) {
	case 0:
		str = "Wrong descriptor format/order";
		break;
	case 1:
		str = "Descriptor fetch failed";
		break;
	case 2:
		str = "Tail descriptor not EOP/NOP";
		break;
	case 3:
		str = "False scheduling error";
		break;
	case 4:
		str = "Tail value larger than ring len";
		break;
	case 5:
		str = "Too many data commands";
		break;
	case 6:
		str = "Zero packets sent in quanta";
		break;
	case 7:
		str = "Packet too small or too big";
		break;
	case 8:
		str = "TSO length doesn't match sum";
		break;
	case 9:
		str = "TSO tail reached before TLEN";
		break;
	case 10:
		str = "TSO max 3 descs for headers";
		break;
	case 11:
		str = "EOP on header descriptor";
		break;
	case 12:
		str = "MSS is 0 or TLEN is 0";
		break;
	case 13:
		str = "CTX desc invalid IPSec fields";
		break;
	case 14:
		str = "Quanta invalid # of SSO packets";
		break;
	case 15:
		str = "Quanta bytes exceeds pkt_len*64";
		break;
	case 16:
		str = "Quanta exceeds max_cmds_in_sq";
		break;
	case 17:
		str = "incoherent last_lso_quanta";
		break;
	case 18:
		str = "incoherent TSO TLEN";
		break;
	case 19:
		str = "Quanta: too many descriptors";
		break;
	case 20:
		str = "Quanta: # of packets mismatch";
		break;
	default:
		break;
	}

	if (str)
		snprintf(buf, ICE_STR_BUF_LEN, "%s", str);
	else {
		snprintf(buf, ICE_STR_BUF_LEN,
		    "Unknown Tx TCLAN event %u", event);
	}

	return buf;
}

/**
 * ice_mdd_tx_pqm_str - Convert MDD Tx PQM event to a string
 * @event: the MDD event number to convert
 *
 * Convert the Tx PQM event value from the GL_MDET_TX_PQM register into
 * a human readable string for logging of MDD events.
 */
const char *
ice_mdd_tx_pqm_str(uint8_t event)
{
	static char buf[ICE_STR_BUF_LEN];
	const char *str = NULL;

	switch (event) {
	case 0:
		str = "PCI_DUMMY_COMP";
		break;
	case 1:
		str = "PCI_UR_COMP";
		break;
	/* Index 2 is unused */
	case 3:
		str = "RCV_SH_BE_LSO";
		break;
	case 4:
		str = "Q_FL_MNG_EPY_CH";
		break;
	case 5:
		str = "Q_EPY_MNG_FL_CH";
		break;
	case 6:
		str = "LSO_NUMDESCS_ZERO";
		break;
	case 7:
		str = "LSO_LENGTH_ZERO";
		break;
	case 8:
		str = "LSO_MSS_BELOW_MIN";
		break;
	case 9:
		str = "LSO_MSS_ABOVE_MAX";
		break;
	case 10:
		str = "LSO_HDR_SIZE_ZERO";
		break;
	case 11:
		str = "RCV_CNT_BE_LSO";
		break;
	case 12:
		str = "SKIP_ONE_QT_ONLY";
		break;
	case 13:
		str = "LSO_PKTCNT_ZERO";
		break;
	case 14:
		str = "SSO_LENGTH_ZERO";
		break;
	case 15:
		str = "SSO_LENGTH_EXCEED";
		break;
	case 16:
		str = "SSO_PKTCNT_ZERO";
		break;
	case 17:
		str = "SSO_PKTCNT_EXCEED";
		break;
	case 18:
		str = "SSO_NUMDESCS_ZERO";
		break;
	case 19:
		str = "SSO_NUMDESCS_EXCEED";
		break;
	case 20:
		str = "TAIL_GT_RING_LENGTH";
		break;
	case 21:
		str = "RESERVED_DBL_TYPE";
		break;
	case 22:
		str = "ILLEGAL_HEAD_DROP_DBL";
		break;
	case 23:
		str = "LSO_OVER_COMMS_Q";
		break;
	case 24:
		str = "ILLEGAL_VF_QNUM";
		break;
	case 25:
		str = "QTAIL_GT_RING_LENGTH";
		break;
	default:
		break;
	}

	if (str)
		snprintf(buf, ICE_STR_BUF_LEN, "%s", str);
	else {
		snprintf(buf, ICE_STR_BUF_LEN,
		    "Unknown Tx PQM event %u", event);
	}

	return buf;
}

/**
 * ice_mdd_rx_str - Convert MDD Rx queue event to a string
 * @event: the MDD event number to convert
 *
 * Convert the Rx queue event value from the GL_MDET_RX register into a human
 * readable string for logging of MDD events.
 */
const char *
ice_mdd_rx_str(uint8_t event)
{
	static char buf[ICE_STR_BUF_LEN];
	const char *str = NULL;

	switch (event) {
	case 1:
		str = "Descriptor fetch failed";
		break;
	default:
		break;
	}

	if (str)
		snprintf(buf, ICE_STR_BUF_LEN, "%s", str);
	else
		snprintf(buf, ICE_STR_BUF_LEN, "Unknown Rx event %u", event);

	return buf;
}

/* Memory types */
enum ice_memset_type {
	ICE_NONDMA_MEM = 0,
	ICE_DMA_MEM
};

/* Memcpy types */
enum ice_memcpy_type {
	ICE_NONDMA_TO_NONDMA = 0,
	ICE_NONDMA_TO_DMA,
	ICE_DMA_TO_DMA,
	ICE_DMA_TO_NONDMA
};

/*
 * ice_calloc - Allocate an array of elementes
 * @hw: the hardware private structure
 * @count: number of elements to allocate
 * @size: the size of each element
 *
 * Allocate memory for an array of items equal to size. Note that the OS
 * compatibility layer assumes all allocation functions will provide zero'd
 * memory.
 */
static inline void *
ice_calloc(struct ice_hw __unused *hw, size_t count, size_t size)
{
	return mallocarray(count, size, M_DEVBUF, M_ZERO | M_NOWAIT);
}

/*
 * ice_malloc - Allocate memory of a specified size
 * @hw: the hardware private structure
 * @size: the size to allocate
 *
 * Allocates memory of the specified size. Note that the OS compatibility
 * layer assumes that all allocations will provide zero'd memory.
 */
static inline void *
ice_malloc(struct ice_hw __unused *hw, size_t size)
{
	return malloc(size, M_DEVBUF, M_ZERO | M_NOWAIT);
}

/*
 * ice_memdup - Allocate a copy of some other memory
 * @hw: private hardware structure
 * @src: the source to copy from
 * @size: allocation size
 *
 * Allocate memory of the specified size, and copy bytes from the src to fill
 * it. We don't need to zero this memory as we immediately initialize it by
 * copying from the src pointer.
 */
static inline void *
ice_memdup(struct ice_hw __unused *hw, const void *src, size_t size)
{
	void *dst = malloc(size, M_DEVBUF, M_NOWAIT);

	if (dst != NULL)
		memcpy(dst, src, size);

	return dst;
}

/*
 * ice_free - Free previously allocated memory
 * @hw: the hardware private structure
 * @mem: pointer to the memory to free
 *
 * Free memory that was previously allocated by ice_calloc, ice_malloc, or
 * ice_memdup.
 */
static inline void
ice_free(struct ice_hw __unused *hw, void *mem)
{
	free(mem, M_DEVBUF, 0);
}

/**
 * ice_set_ctrlq_len - Configure ctrlq lengths for a device
 * @hw: the device hardware structure
 *
 * Configures the control queues for the given device, setting up the
 * specified lengths, prior to initializing hardware.
 */
void
ice_set_ctrlq_len(struct ice_hw *hw)
{
	hw->adminq.num_rq_entries = ICE_AQ_LEN;
	hw->adminq.num_sq_entries = ICE_AQ_LEN;
	hw->adminq.rq_buf_size = ICE_AQ_MAX_BUF_LEN;
	hw->adminq.sq_buf_size = ICE_AQ_MAX_BUF_LEN;

	hw->mailboxq.num_rq_entries = ICE_MBXQ_LEN;
	hw->mailboxq.num_sq_entries = ICE_MBXQ_LEN;
	hw->mailboxq.rq_buf_size = ICE_MBXQ_MAX_BUF_LEN;
	hw->mailboxq.sq_buf_size = ICE_MBXQ_MAX_BUF_LEN;

}

enum ice_fw_modes
ice_get_fw_mode(struct ice_hw *hw)
{
#define ICE_FW_MODE_DBG_M (1 << 0)
#define ICE_FW_MODE_REC_M (1 << 1)
#define ICE_FW_MODE_ROLLBACK_M (1 << 2)
	uint32_t fw_mode;

	/* check the current FW mode */
	fw_mode = ICE_READ(hw, GL_MNG_FWSM) & GL_MNG_FWSM_FW_MODES_M;
	if (fw_mode & ICE_FW_MODE_DBG_M)
		return ICE_FW_MODE_DBG;
	else if (fw_mode & ICE_FW_MODE_REC_M)
		return ICE_FW_MODE_REC;
	else if (fw_mode & ICE_FW_MODE_ROLLBACK_M)
		return ICE_FW_MODE_ROLLBACK;
	else
		return ICE_FW_MODE_NORMAL;
}

void
ice_set_mac_type(struct ice_hw *hw)
{
	struct ice_softc *sc = hw->hw_sc;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);

	switch (sc->sc_pid) {
#if 0
	case ICE_DEV_ID_E810C_BACKPLANE:
	case ICE_DEV_ID_E810C_QSFP:
	case ICE_DEV_ID_E810C_SFP:
	case ICE_DEV_ID_E810_XXV_BACKPLANE:
	case ICE_DEV_ID_E810_XXV_QSFP:
	case ICE_DEV_ID_E810_XXV_SFP:
#endif
	case PCI_PRODUCT_INTEL_E810_XXV_SFP:
		hw->mac_type = ICE_MAC_E810;
		break;
#if 0
	case ICE_DEV_ID_E822C_10G_BASE_T:
	case ICE_DEV_ID_E822C_BACKPLANE:
	case ICE_DEV_ID_E822C_QSFP:
	case ICE_DEV_ID_E822C_SFP:
	case ICE_DEV_ID_E822C_SGMII:
	case ICE_DEV_ID_E822L_10G_BASE_T:
	case ICE_DEV_ID_E822L_BACKPLANE:
	case ICE_DEV_ID_E822L_SFP:
	case ICE_DEV_ID_E822L_SGMII:
	case ICE_DEV_ID_E823L_10G_BASE_T:
	case ICE_DEV_ID_E823L_1GBE:
	case ICE_DEV_ID_E823L_BACKPLANE:
	case ICE_DEV_ID_E823L_QSFP:
	case ICE_DEV_ID_E823L_SFP:
	case ICE_DEV_ID_E823C_10G_BASE_T:
	case ICE_DEV_ID_E823C_BACKPLANE:
	case ICE_DEV_ID_E823C_QSFP:
	case ICE_DEV_ID_E823C_SFP:
	case ICE_DEV_ID_E823C_SGMII:
		hw->mac_type = ICE_MAC_GENERIC;
		break;
#endif
	default:
		hw->mac_type = ICE_MAC_UNKNOWN;
		break;
	}

	DNPRINTF(ICE_DBG_INIT, "mac_type: %d\n", hw->mac_type);
}

enum ice_status
ice_check_reset(struct ice_hw *hw)
{
	uint32_t cnt, reg = 0, grst_timeout, uld_mask, reset_wait_cnt;

	/* Poll for Device Active state in case a recent CORER, GLOBR,
	 * or EMPR has occurred. The grst delay value is in 100ms units.
	 * Add 1sec for outstanding AQ commands that can take a long time.
	 */
	grst_timeout = ((ICE_READ(hw, GLGEN_RSTCTL) & GLGEN_RSTCTL_GRSTDEL_M) >>
			GLGEN_RSTCTL_GRSTDEL_S) + 10;

	for (cnt = 0; cnt < grst_timeout; cnt++) {
		ice_msec_delay(100, true);
		reg = ICE_READ(hw, GLGEN_RSTAT);
		if (!(reg & GLGEN_RSTAT_DEVSTATE_M))
			break;
	}

	if (cnt == grst_timeout) {
		DNPRINTF(ICE_DBG_INIT, "Global reset polling failed to complete.\n");
		return ICE_ERR_RESET_FAILED;
	}

#define ICE_RESET_DONE_MASK	(GLNVM_ULD_PCIER_DONE_M |\
				 GLNVM_ULD_PCIER_DONE_1_M |\
				 GLNVM_ULD_CORER_DONE_M |\
				 GLNVM_ULD_GLOBR_DONE_M |\
				 GLNVM_ULD_POR_DONE_M |\
				 GLNVM_ULD_POR_DONE_1_M |\
				 GLNVM_ULD_PCIER_DONE_2_M)

	uld_mask = ICE_RESET_DONE_MASK | (hw->func_caps.common_cap.iwarp ?
					  GLNVM_ULD_PE_DONE_M : 0);

	reset_wait_cnt = ICE_PF_RESET_WAIT_COUNT;

	/* Device is Active; check Global Reset processes are done */
	for (cnt = 0; cnt < reset_wait_cnt; cnt++) {
		reg = ICE_READ(hw, GLNVM_ULD) & uld_mask;
		if (reg == uld_mask) {
			DNPRINTF(ICE_DBG_INIT, "Global reset processes done. %d\n", cnt);
			break;
		}
		ice_msec_delay(10, true);
	}

	if (cnt == reset_wait_cnt) {
		DNPRINTF(ICE_DBG_INIT, "Wait for Reset Done timed out. "
		    "GLNVM_ULD = 0x%x\n", reg);
		return ICE_ERR_RESET_FAILED;
	}

	return ICE_SUCCESS;
}

/*
 * ice_pf_reset - Reset the PF
 *
 * If a global reset has been triggered, this function checks
 * for its completion and then issues the PF reset
 */
enum ice_status
ice_pf_reset(struct ice_hw *hw)
{
	uint32_t cnt, reg, reset_wait_cnt, cfg_lock_timeout;

	/* If at function entry a global reset was already in progress, i.e.
	 * state is not 'device active' or any of the reset done bits are not
	 * set in GLNVM_ULD, there is no need for a PF Reset; poll until the
	 * global reset is done.
	 */
	if ((ICE_READ(hw, GLGEN_RSTAT) & GLGEN_RSTAT_DEVSTATE_M) ||
	    (ICE_READ(hw, GLNVM_ULD) & ICE_RESET_DONE_MASK) ^ ICE_RESET_DONE_MASK) {
		/* poll on global reset currently in progress until done */
		if (ice_check_reset(hw))
			return ICE_ERR_RESET_FAILED;

		return ICE_SUCCESS;
	}

	/* Reset the PF */
	reg = ICE_READ(hw, PFGEN_CTRL);

	ICE_WRITE(hw, PFGEN_CTRL, (reg | PFGEN_CTRL_PFSWR_M));

	/* Wait for the PFR to complete. The wait time is the global config lock
	 * timeout plus the PFR timeout which will account for a possible reset
	 * that is occurring during a download package operation.
	 */
	reset_wait_cnt = ICE_PF_RESET_WAIT_COUNT;
	cfg_lock_timeout = ICE_GLOBAL_CFG_LOCK_TIMEOUT;

	for (cnt = 0; cnt < cfg_lock_timeout + reset_wait_cnt; cnt++) {
		reg = ICE_READ(hw, PFGEN_CTRL);
		if (!(reg & PFGEN_CTRL_PFSWR_M))
			break;

		ice_msec_delay(1, true);
	}

	if (cnt == cfg_lock_timeout + reset_wait_cnt) {
		DNPRINTF(ICE_DBG_INIT, "PF reset polling failed to complete.\n");
		return ICE_ERR_RESET_FAILED;
	}

	return ICE_SUCCESS;
}

/**
 * ice_reset - Perform different types of reset
 *
 * If anything other than a PF reset is triggered, PXE mode is restored.
 * This has to be cleared using ice_clear_pxe_mode again, once the AQ
 * interface has been restored in the rebuild flow.
 */
enum ice_status
ice_reset(struct ice_hw *hw, enum ice_reset_req req)
{
	uint32_t val = 0;

	switch (req) {
	case ICE_RESET_PFR:
		return ice_pf_reset(hw);
	case ICE_RESET_CORER:
		DNPRINTF(ICE_DBG_INIT, "CoreR requested\n");
		val = GLGEN_RTRIG_CORER_M;
		break;
	case ICE_RESET_GLOBR:
		DNPRINTF(ICE_DBG_INIT, "GlobalR requested\n");
		val = GLGEN_RTRIG_GLOBR_M;
		break;
	default:
		return ICE_ERR_PARAM;
	}

	val |= ICE_READ(hw, GLGEN_RTRIG);
	ICE_WRITE(hw, GLGEN_RTRIG, val);
	ice_flush(hw);

	/* wait for the FW to be ready */
	return ice_check_reset(hw);
}

/*
 * ice_get_itr_intrl_gran
 *
 * Determines the ITR/INTRL granularities based on the maximum aggregate
 * bandwidth according to the device's configuration during power-on.
 */
void
ice_get_itr_intrl_gran(struct ice_hw *hw)
{
	uint8_t max_agg_bw = (ICE_READ(hw, GL_PWR_MODE_CTL) &
			 GL_PWR_MODE_CTL_CAR_MAX_BW_M) >>
			GL_PWR_MODE_CTL_CAR_MAX_BW_S;

	switch (max_agg_bw) {
	case ICE_MAX_AGG_BW_200G:
	case ICE_MAX_AGG_BW_100G:
	case ICE_MAX_AGG_BW_50G:
		hw->itr_gran = ICE_ITR_GRAN_ABOVE_25;
		hw->intrl_gran = ICE_INTRL_GRAN_ABOVE_25;
		break;
	case ICE_MAX_AGG_BW_25G:
		hw->itr_gran = ICE_ITR_GRAN_MAX_25;
		hw->intrl_gran = ICE_INTRL_GRAN_MAX_25;
		break;
	}
}

/*
 * ice_destroy_ctrlq_locks - Destroy locks for a control queue
 * @cq: pointer to the control queue
 *
 * Destroys the send and receive queue locks for a given control queue.
 */
void
ice_destroy_ctrlq_locks(struct ice_ctl_q_info *cq)
{
#if 0
	ice_destroy_lock(&cq->sq_lock);
	ice_destroy_lock(&cq->rq_lock);
#endif
}

/* Returns true if Queue is enabled else false. */
bool
ice_check_sq_alive(struct ice_hw *hw, struct ice_ctl_q_info *cq)
{
	/* check both queue-length and queue-enable fields */
	if (cq->sq.len && cq->sq.len_mask && cq->sq.len_ena_mask)
		return (ICE_READ(hw, cq->sq.len) & (cq->sq.len_mask |
						cq->sq.len_ena_mask)) ==
			(cq->num_sq_entries | cq->sq.len_ena_mask);

	return false;
}

#define ICE_CQ_INIT_REGS(qinfo, prefix)				\
do {								\
	(qinfo)->sq.head = prefix##_ATQH;			\
	(qinfo)->sq.tail = prefix##_ATQT;			\
	(qinfo)->sq.len = prefix##_ATQLEN;			\
	(qinfo)->sq.bah = prefix##_ATQBAH;			\
	(qinfo)->sq.bal = prefix##_ATQBAL;			\
	(qinfo)->sq.len_mask = prefix##_ATQLEN_ATQLEN_M;	\
	(qinfo)->sq.len_ena_mask = prefix##_ATQLEN_ATQENABLE_M;	\
	(qinfo)->sq.len_crit_mask = prefix##_ATQLEN_ATQCRIT_M;	\
	(qinfo)->sq.head_mask = prefix##_ATQH_ATQH_M;		\
	(qinfo)->rq.head = prefix##_ARQH;			\
	(qinfo)->rq.tail = prefix##_ARQT;			\
	(qinfo)->rq.len = prefix##_ARQLEN;			\
	(qinfo)->rq.bah = prefix##_ARQBAH;			\
	(qinfo)->rq.bal = prefix##_ARQBAL;			\
	(qinfo)->rq.len_mask = prefix##_ARQLEN_ARQLEN_M;	\
	(qinfo)->rq.len_ena_mask = prefix##_ARQLEN_ARQENABLE_M;	\
	(qinfo)->rq.len_crit_mask = prefix##_ARQLEN_ARQCRIT_M;	\
	(qinfo)->rq.head_mask = prefix##_ARQH_ARQH_M;		\
} while (0)

/*
 * ice_adminq_init_regs - Initialize AdminQ registers
 *
 * This assumes the alloc_sq and alloc_rq functions have already been called
 */
void
ice_adminq_init_regs(struct ice_hw *hw)
{
	struct ice_ctl_q_info *cq = &hw->adminq;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);

	ICE_CQ_INIT_REGS(cq, PF_FW);
}

/*
 * ice_mailbox_init_regs - Initialize Mailbox registers
 *
 * This assumes the alloc_sq and alloc_rq functions have already been called
 */
void
ice_mailbox_init_regs(struct ice_hw *hw)
{
	struct ice_ctl_q_info *cq = &hw->mailboxq;

	ICE_CQ_INIT_REGS(cq, PF_MBX);
}

/*
 * ice_free_dma_mem - Free DMA memory allocated by ice_alloc_dma_mem
 * @hw: the hardware private structure
 * @mem: DMA memory to free
 *
 * Release the bus DMA tag and map, and free the DMA memory associated with
 * it.
 */
void
ice_free_dma_mem(struct ice_hw __unused *hw, struct ice_dma_mem *mem)
{
	if (mem->map != NULL) {
		if (mem->va != NULL) {
			bus_dmamap_sync(mem->tag, mem->map, 0, mem->size,
			    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
			bus_dmamap_unload(mem->tag, mem->map);
			bus_dmamem_unmap(mem->tag, mem->va, mem->size);
			bus_dmamem_free(mem->tag, &mem->seg, 1);
			mem->va = NULL;
		}

		bus_dmamap_destroy(mem->tag, mem->map);
		mem->map = NULL;
		mem->pa = 0L;
	}
}

/* ice_alloc_dma_mem - Request OS to allocate DMA memory */
void *
ice_alloc_dma_mem(struct ice_hw *hw, struct ice_dma_mem *mem, uint64_t size)
{
	struct ice_softc *sc = hw->hw_sc;
	int nsegs = 0, err;
	caddr_t va;

	mem->tag = sc->sc_dmat;

	err = bus_dmamap_create(mem->tag, size, 1, size, 0, BUS_DMA_NOWAIT,
	    &mem->map);
	if (err)
		goto fail;

	err = bus_dmamem_alloc(mem->tag, size, 1, 0, &mem->seg, 1, &nsegs,
	    BUS_DMA_NOWAIT | BUS_DMA_ZERO);
	if (err || nsegs != 1)
		goto fail_1;

	err = bus_dmamem_map(mem->tag, &mem->seg, nsegs, size, &va,
	    BUS_DMA_NOWAIT | BUS_DMA_COHERENT);
	if (err)
		goto fail_2;

	mem->va = va;
	mem->size = size;

	err = bus_dmamap_load(mem->tag, mem->map, mem->va, size, NULL,
	    BUS_DMA_NOWAIT);
	if (err)
		goto fail_3;

	bus_dmamap_sync(mem->tag, mem->map, 0, size, BUS_DMASYNC_PREWRITE);
	mem->pa = mem->map->dm_segs[0].ds_addr;
	return (mem->va);
fail_3:
	bus_dmamem_unmap(mem->tag, mem->va, size);
fail_2:
	bus_dmamem_free(mem->tag, &mem->seg, nsegs);
fail_1:
	bus_dmamap_destroy(mem->tag, mem->map);
fail:
	mem->map = NULL;
	mem->tag = NULL;
	mem->va = NULL;
	mem->pa = 0L;
	mem->size = 0;
	return (NULL);
}

/* ice_alloc_ctrlq_sq_ring - Allocate Control Transmit Queue (ATQ) rings */
enum ice_status
ice_alloc_ctrlq_sq_ring(struct ice_hw *hw, struct ice_ctl_q_info *cq)
{
	size_t size = cq->num_sq_entries * sizeof(struct ice_aq_desc);

	cq->sq.desc_buf.va = ice_alloc_dma_mem(hw, &cq->sq.desc_buf, size);
	if (!cq->sq.desc_buf.va)
		return ICE_ERR_NO_MEMORY;

	return ICE_SUCCESS;
}

/* ice_alloc_ctrlq_rq_ring - Allocate Control Receive Queue (ARQ) rings */
enum ice_status
ice_alloc_ctrlq_rq_ring(struct ice_hw *hw, struct ice_ctl_q_info *cq)
{
	size_t size = cq->num_rq_entries * sizeof(struct ice_aq_desc);

	cq->rq.desc_buf.va = ice_alloc_dma_mem(hw, &cq->rq.desc_buf, size);
	if (!cq->rq.desc_buf.va)
		return ICE_ERR_NO_MEMORY;

	return ICE_SUCCESS;
}

/* ice_alloc_sq_bufs - Allocate empty buffer structs for the ATQ */
enum ice_status
ice_alloc_sq_bufs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
{
	int i;

	/* No mapped memory needed yet, just the buffer info structures */
	cq->sq.dma_head = ice_calloc(hw, cq->num_sq_entries,
				     sizeof(cq->sq.desc_buf));
	if (!cq->sq.dma_head)
		return ICE_ERR_NO_MEMORY;
	cq->sq.r.sq_bi = (struct ice_dma_mem *)cq->sq.dma_head;

	/* allocate the mapped buffers */
	for (i = 0; i < cq->num_sq_entries; i++) {
		struct ice_dma_mem *bi;

		bi = &cq->sq.r.sq_bi[i];
		bi->va = ice_alloc_dma_mem(hw, bi, cq->sq_buf_size);
		if (!bi->va)
			goto unwind_alloc_sq_bufs;
	}
	return ICE_SUCCESS;

unwind_alloc_sq_bufs:
	/* don't try to free the one that failed... */
	i--;
	for (; i >= 0; i--)
		ice_free_dma_mem(hw, &cq->sq.r.sq_bi[i]);
	cq->sq.r.sq_bi = NULL;
	ice_free(hw, cq->sq.dma_head);
	cq->sq.dma_head = NULL;

	return ICE_ERR_NO_MEMORY;
}

/*
 * ice_free_cq_ring - Free control queue ring
 * @hw: pointer to the hardware structure
 * @ring: pointer to the specific control queue ring
 *
 * This assumes the posted buffers have already been cleaned
 * and de-allocated
 */
void
ice_free_cq_ring(struct ice_hw *hw, struct ice_ctl_q_ring *ring)
{
	ice_free_dma_mem(hw, &ring->desc_buf);
}

/* ice_alloc_rq_bufs - Allocate pre-posted buffers for the ARQ */
enum ice_status
ice_alloc_rq_bufs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
{
	int i;

	/*
	 * We'll be allocating the buffer info memory first, then we can
	 * allocate the mapped buffers for the event processing
	 */
	cq->rq.dma_head = ice_calloc(hw, cq->num_rq_entries,
				     sizeof(cq->rq.desc_buf));
	if (!cq->rq.dma_head)
		return ICE_ERR_NO_MEMORY;

	cq->rq.r.rq_bi = (struct ice_dma_mem *)cq->rq.dma_head;

	/* allocate the mapped buffers */
	for (i = 0; i < cq->num_rq_entries; i++) {
		struct ice_aq_desc *desc;
		struct ice_dma_mem *bi;

		bi = &cq->rq.r.rq_bi[i];
		bi->va = ice_alloc_dma_mem(hw, bi, cq->rq_buf_size);
		if (!bi->va)
			goto unwind_alloc_rq_bufs;

		/* now configure the descriptors for use */
		desc = ICE_CTL_Q_DESC(cq->rq, i);

		desc->flags = htole16(ICE_AQ_FLAG_BUF);
		if (cq->rq_buf_size > ICE_AQ_LG_BUF)
			desc->flags |= htole16(ICE_AQ_FLAG_LB);
		desc->opcode = 0;
		/* This is in accordance with control queue design, there is no
		 * register for buffer size configuration
		 */
		desc->datalen = htole16(bi->size);
		desc->retval = 0;
		desc->cookie_high = 0;
		desc->cookie_low = 0;
		desc->params.generic.addr_high = htole32(ICE_HI_DWORD(bi->pa));
		desc->params.generic.addr_low = htole32(ICE_LO_DWORD(bi->pa));
		desc->params.generic.param0 = 0;
		desc->params.generic.param1 = 0;
	}
	return ICE_SUCCESS;

unwind_alloc_rq_bufs:
	/* don't try to free the one that failed... */
	i--;
	for (; i >= 0; i--)
		ice_free_dma_mem(hw, &cq->rq.r.rq_bi[i]);
	cq->rq.r.rq_bi = NULL;
	ice_free(hw, cq->rq.dma_head);
	cq->rq.dma_head = NULL;

	return ICE_ERR_NO_MEMORY;
}

enum ice_status
ice_cfg_cq_regs(struct ice_hw *hw, struct ice_ctl_q_ring *ring,
    uint16_t num_entries)
{
	/* Clear Head and Tail */
	ICE_WRITE(hw, ring->head, 0);
	ICE_WRITE(hw, ring->tail, 0);

	/* set starting point */
	ICE_WRITE(hw, ring->len, (num_entries | ring->len_ena_mask));
	ICE_WRITE(hw, ring->bal, ICE_LO_DWORD(ring->desc_buf.pa));
	ICE_WRITE(hw, ring->bah, ICE_HI_DWORD(ring->desc_buf.pa));

	/* Check one register to verify that config was applied */
	if (ICE_READ(hw, ring->bal) != ICE_LO_DWORD(ring->desc_buf.pa))
		return ICE_ERR_AQ_ERROR;

	return ICE_SUCCESS;
}

/**
 * ice_cfg_sq_regs - configure Control ATQ registers
 * @hw: pointer to the hardware structure
 * @cq: pointer to the specific Control queue
 *
 * Configure base address and length registers for the transmit queue
 */
enum ice_status
ice_cfg_sq_regs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
{
	return ice_cfg_cq_regs(hw, &cq->sq, cq->num_sq_entries);
}

/*
 * ice_cfg_rq_regs - configure Control ARQ register
 * Configure base address and length registers for the receive (event queue)
 */
enum ice_status
ice_cfg_rq_regs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
{
	enum ice_status status;

	status = ice_cfg_cq_regs(hw, &cq->rq, cq->num_rq_entries);
	if (status)
		return status;

	/* Update tail in the HW to post pre-allocated buffers */
	ICE_WRITE(hw, cq->rq.tail, (uint32_t)(cq->num_rq_entries - 1));

	return ICE_SUCCESS;
}

#define ICE_FREE_CQ_BUFS(hw, qi, ring)					\
do {									\
	/* free descriptors */						\
	if ((qi)->ring.r.ring##_bi) {					\
		int i;							\
									\
		for (i = 0; i < (qi)->num_##ring##_entries; i++)	\
			if ((qi)->ring.r.ring##_bi[i].pa)		\
				ice_free_dma_mem((hw),			\
					&(qi)->ring.r.ring##_bi[i]);	\
	}								\
	/* free DMA head */						\
	ice_free(hw, (qi)->ring.dma_head);				\
} while (0)

/*
 * ice_init_sq - main initialization routine for Control ATQ
 * @hw: pointer to the hardware structure
 * @cq: pointer to the specific Control queue
 *
 * This is the main initialization routine for the Control Send Queue
 * Prior to calling this function, the driver *MUST* set the following fields
 * in the cq->structure:
 *     - cq->num_sq_entries
 *     - cq->sq_buf_size
 *
 * Do *NOT* hold the lock when calling this as the memory allocation routines
 * called are not going to be atomic context safe
 */
enum ice_status
ice_init_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
{
	enum ice_status ret_code;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);

	if (cq->sq.count > 0) {
		/* queue already initialized */
		ret_code = ICE_ERR_NOT_READY;
		goto init_ctrlq_exit;
	}

	/* verify input for valid configuration */
	if (!cq->num_sq_entries || !cq->sq_buf_size) {
		ret_code = ICE_ERR_CFG;
		goto init_ctrlq_exit;
	}

	cq->sq.next_to_use = 0;
	cq->sq.next_to_clean = 0;

	/* allocate the ring memory */
	ret_code = ice_alloc_ctrlq_sq_ring(hw, cq);
	if (ret_code)
		goto init_ctrlq_exit;

	/* allocate buffers in the rings */
	ret_code = ice_alloc_sq_bufs(hw, cq);
	if (ret_code)
		goto init_ctrlq_free_rings;

	/* initialize base registers */
	ret_code = ice_cfg_sq_regs(hw, cq);
	if (ret_code)
		goto init_ctrlq_free_rings;

	/* success! */
	cq->sq.count = cq->num_sq_entries;
	return ICE_SUCCESS;

init_ctrlq_free_rings:
	ICE_FREE_CQ_BUFS(hw, cq, sq);
	ice_free_cq_ring(hw, &cq->sq);
init_ctrlq_exit:
	return ret_code;
}

/*
 * ice_init_rq - initialize receive side of a control queue
 * @hw: pointer to the hardware structure
 * @cq: pointer to the specific Control queue
 *
 * The main initialization routine for Receive side of a control queue.
 * Prior to calling this function, the driver *MUST* set the following fields
 * in the cq->structure:
 *     - cq->num_rq_entries
 *     - cq->rq_buf_size
 *
 * Do *NOT* hold the lock when calling this as the memory allocation routines
 * called are not going to be atomic context safe
 */
enum ice_status
ice_init_rq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
{
	enum ice_status ret_code;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);

	if (cq->rq.count > 0) {
		/* queue already initialized */
		ret_code = ICE_ERR_NOT_READY;
		goto init_ctrlq_exit;
	}

	/* verify input for valid configuration */
	if (!cq->num_rq_entries || !cq->rq_buf_size) {
		ret_code = ICE_ERR_CFG;
		goto init_ctrlq_exit;
	}

	cq->rq.next_to_use = 0;
	cq->rq.next_to_clean = 0;

	/* allocate the ring memory */
	ret_code = ice_alloc_ctrlq_rq_ring(hw, cq);
	if (ret_code)
		goto init_ctrlq_exit;

	/* allocate buffers in the rings */
	ret_code = ice_alloc_rq_bufs(hw, cq);
	if (ret_code)
		goto init_ctrlq_free_rings;

	/* initialize base registers */
	ret_code = ice_cfg_rq_regs(hw, cq);
	if (ret_code)
		goto init_ctrlq_free_rings;

	/* success! */
	cq->rq.count = cq->num_rq_entries;
	goto init_ctrlq_exit;

init_ctrlq_free_rings:
	ICE_FREE_CQ_BUFS(hw, cq, rq);
	ice_free_cq_ring(hw, &cq->rq);
init_ctrlq_exit:
	return ret_code;
}

/*
 * Decide if we should retry the send command routine for the ATQ, depending
 * on the opcode.
 */
bool
ice_should_retry_sq_send_cmd(uint16_t opcode)
{
	switch (opcode) {
	case ice_aqc_opc_dnl_get_status:
	case ice_aqc_opc_dnl_run:
	case ice_aqc_opc_dnl_call:
	case ice_aqc_opc_dnl_read_sto:
	case ice_aqc_opc_dnl_write_sto:
	case ice_aqc_opc_dnl_set_breakpoints:
	case ice_aqc_opc_dnl_read_log:
	case ice_aqc_opc_get_link_topo:
	case ice_aqc_opc_done_alt_write:
	case ice_aqc_opc_lldp_stop:
	case ice_aqc_opc_lldp_start:
	case ice_aqc_opc_lldp_filter_ctrl:
		return true;
	}

	return false;
}

/* based on libsa/hexdump.c */
void
ice_hexdump(const void *addr, size_t size)
{
	const unsigned char *line, *end;
	int byte;

	end = (const char *)addr + size;
	for (line = addr; line < end; line += 16) {
		DPRINTF("%08lx  ", (unsigned long)line);
		for (byte = 0; byte < 16; byte++) {
			if (&line[byte] < end)
				DPRINTF("%02x ", line[byte]);
			else
				DPRINTF("   ");
			if (byte == 7)
				DPRINTF(" ");
		}
		DPRINTF(" |");
		for (byte = 0; byte < 16; byte++) {
			if (&line[byte] < end) {
				if (line[byte] >= ' ' && line[byte] <= '~')
					DPRINTF("%c", line[byte]);
				else
					DPRINTF(".");
			} else
				break;
		}
		DPRINTF("|\n");
	}
	DPRINTF("%08lx\n", (unsigned long)end);
}

/**
 * ice_debug_array - Format and print an array of values to the console
 * @hw: private hardware structure
 * @mask: the debug message type
 * @groupsize: preferred size in bytes to print each chunk
 * @buf: the array buffer to print
 * @len: size of the array buffer
 *
 * Format the given array as a series of uint8_t values with hexadecimal
 * notation and log the contents to the console log.
 *
 * TODO: Currently only supports a group size of 1, due to the way hexdump is
 * implemented.
 */
void
ice_debug_array(struct ice_hw *hw, uint64_t mask, uint32_t rowsize,
		uint32_t __unused groupsize, uint8_t *buf, size_t len)
{
	if (!(mask & hw->debug_mask))
		return;

	/* Make sure the row-size isn't too large */
	if (rowsize > 0xFF)
		rowsize = 0xFF;

	ice_hexdump(buf, len);
}

/**
 * ice_clean_sq - cleans send side of a control queue
 * @hw: pointer to the hardware structure
 * @cq: pointer to the specific Control queue
 *
 * returns the number of free desc
 */
uint16_t
ice_clean_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
{
	struct ice_ctl_q_ring *sq = &cq->sq;
	uint16_t ntc = sq->next_to_clean;
	struct ice_aq_desc *desc;

	desc = ICE_CTL_Q_DESC(*sq, ntc);

	while (ICE_READ(hw, cq->sq.head) != ntc) {
		DNPRINTF(ICE_DBG_AQ_MSG, "ntc %d head %d.\n", ntc,
		    ICE_READ(hw, cq->sq.head));
		memset(desc, 0, sizeof(*desc));
		ntc++;
		if (ntc == sq->count)
			ntc = 0;
		desc = ICE_CTL_Q_DESC(*sq, ntc);
	}

	sq->next_to_clean = ntc;

	return ICE_CTL_Q_DESC_UNUSED(sq);
}

/**
 * ice_ctl_q_str - Convert control queue type to string
 * @qtype: the control queue type
 *
 * Returns: A string name for the given control queue type.
 */
const char *
ice_ctl_q_str(enum ice_ctl_q qtype)
{
	switch (qtype) {
	case ICE_CTL_Q_UNKNOWN:
		return "Unknown CQ";
	case ICE_CTL_Q_ADMIN:
		return "AQ";
	case ICE_CTL_Q_MAILBOX:
		return "MBXQ";
	default:
		return "Unrecognized CQ";
	}
}

/**
 * ice_debug_cq
 * @hw: pointer to the hardware structure
 * @cq: pointer to the specific Control queue
 * @desc: pointer to control queue descriptor
 * @buf: pointer to command buffer
 * @buf_len: max length of buf
 * @response: true if this is the writeback response
 *
 * Dumps debug log about control command with descriptor contents.
 */
void
ice_debug_cq(struct ice_hw *hw, struct ice_ctl_q_info *cq,
	     void *desc, void *buf, uint16_t buf_len, bool response)
{
#ifdef ICE_DEBUG
	struct ice_aq_desc *cq_desc = (struct ice_aq_desc *)desc;
	uint16_t datalen, flags;

	if (!((ICE_DBG_AQ_DESC | ICE_DBG_AQ_DESC_BUF) & hw->debug_mask))
		return;

	if (!desc)
		return;

	datalen = le16toh(cq_desc->datalen);
	flags = le16toh(cq_desc->flags);

	DNPRINTF(ICE_DBG_AQ_DESC, "%s %s: opcode 0x%04X, flags 0x%04X, "
	    "datalen 0x%04X, retval 0x%04X\n", ice_ctl_q_str(cq->qtype),
	    response ? "Response" : "Command", le16toh(cq_desc->opcode),
	    flags, datalen, le16toh(cq_desc->retval));
	DNPRINTF(ICE_DBG_AQ_DESC, "\tcookie (h,l) 0x%08X 0x%08X\n",
		  le32toh(cq_desc->cookie_high),
		  le32toh(cq_desc->cookie_low));
	DNPRINTF(ICE_DBG_AQ_DESC, "\tparam (0,1)  0x%08X 0x%08X\n",
		  le32toh(cq_desc->params.generic.param0),
		  le32toh(cq_desc->params.generic.param1));
	DNPRINTF(ICE_DBG_AQ_DESC, "\taddr (h,l)   0x%08X 0x%08X\n",
		  le32toh(cq_desc->params.generic.addr_high),
		  le32toh(cq_desc->params.generic.addr_low));
	/* Dump buffer iff 1) one exists and 2) is either a response indicated
	 * by the DD and/or CMP flag set or a command with the RD flag set.
	 */
	if (buf && cq_desc->datalen != 0 &&
	    (flags & (ICE_AQ_FLAG_DD | ICE_AQ_FLAG_CMP) ||
	     flags & ICE_AQ_FLAG_RD)) {
		DNPRINTF(ICE_DBG_AQ_DESC_BUF, "Buffer:\n");
		ice_debug_array(hw, ICE_DBG_AQ_DESC_BUF, 16, 1, (uint8_t *)buf,
		    MIN(buf_len, datalen));
	}
#endif
}

/*
 * ice_sq_done - check if the last send on a control queue has completed
 *
 * Returns: true if all the descriptors on the send side of a control queue
 *          are finished processing, false otherwise.
 */
bool
ice_sq_done(struct ice_hw *hw, struct ice_ctl_q_info *cq)
{
	/* control queue designers suggest use of head for better
	 * timing reliability than DD bit
	 */
	return ICE_READ(hw, cq->sq.head) == cq->sq.next_to_use;
}

/**
 * ice_sq_send_cmd_nolock - send command to a control queue
 * @hw: pointer to the HW struct
 * @cq: pointer to the specific Control queue
 * @desc: prefilled descriptor describing the command (non DMA mem)
 * @buf: buffer to use for indirect commands (or NULL for direct commands)
 * @buf_size: size of buffer for indirect commands (or 0 for direct commands)
 * @cd: pointer to command details structure
 *
 * This is the main send command routine for a control queue. It prepares the
 * command into a descriptor, bumps the send queue tail, waits for the command
 * to complete, captures status and data for the command, etc.
 */
enum ice_status
ice_sq_send_cmd_nolock(struct ice_hw *hw, struct ice_ctl_q_info *cq,
    struct ice_aq_desc *desc, void *buf, uint16_t buf_size,
    struct ice_sq_cd *cd)
{
	struct ice_dma_mem *dma_buf = NULL;
	struct ice_aq_desc *desc_on_ring;
	bool cmd_completed = false;
	enum ice_status status = ICE_SUCCESS;
	uint32_t total_delay = 0;
	uint16_t retval = 0;
	uint32_t val = 0;

	/* if reset is in progress return a soft error */
	if (hw->reset_ongoing)
		return ICE_ERR_RESET_ONGOING;

	cq->sq_last_status = ICE_AQ_RC_OK;

	if (!cq->sq.count) {
		DNPRINTF(ICE_DBG_AQ_MSG,
		    "Control Send queue not initialized.\n");
		status = ICE_ERR_AQ_EMPTY;
		goto sq_send_command_error;
	}

	if ((buf && !buf_size) || (!buf && buf_size)) {
		status = ICE_ERR_PARAM;
		goto sq_send_command_error;
	}

	if (buf) {
		if (buf_size > cq->sq_buf_size) {
			DNPRINTF(ICE_DBG_AQ_MSG,
			    "Invalid buffer size for Control Send queue: %d.\n",
			    buf_size);
			status = ICE_ERR_INVAL_SIZE;
			goto sq_send_command_error;
		}

		desc->flags |= htole16(ICE_AQ_FLAG_BUF);
		if (buf_size > ICE_AQ_LG_BUF)
			desc->flags |= htole16(ICE_AQ_FLAG_LB);
	}

	val = ICE_READ(hw, cq->sq.head);
	if (val >= cq->num_sq_entries) {
		DNPRINTF(ICE_DBG_AQ_MSG,
		    "head overrun at %d in the Control Send Queue ring\n", val);
		status = ICE_ERR_AQ_EMPTY;
		goto sq_send_command_error;
	}

	/* Call clean and check queue available function to reclaim the
	 * descriptors that were processed by FW/MBX; the function returns the
	 * number of desc available. The clean function called here could be
	 * called in a separate thread in case of asynchronous completions.
	 */
	if (ice_clean_sq(hw, cq) == 0) {
		DNPRINTF(ICE_DBG_AQ_MSG,
		    "Error: Control Send Queue is full.\n");
		status = ICE_ERR_AQ_FULL;
		goto sq_send_command_error;
	}

	/* initialize the temp desc pointer with the right desc */
	desc_on_ring = ICE_CTL_Q_DESC(cq->sq, cq->sq.next_to_use);

	/* if the desc is available copy the temp desc to the right place */
	memcpy(desc_on_ring, desc, sizeof(*desc_on_ring));

	/* if buf is not NULL assume indirect command */
	if (buf) {
		dma_buf = &cq->sq.r.sq_bi[cq->sq.next_to_use];
		/* copy the user buf into the respective DMA buf */
		memcpy(dma_buf->va, buf, buf_size);
		desc_on_ring->datalen = htole16(buf_size);

		/* Update the address values in the desc with the pa value
		 * for respective buffer
		 */
		desc_on_ring->params.generic.addr_high =
			htole32(ICE_HI_DWORD(dma_buf->pa));
		desc_on_ring->params.generic.addr_low =
			htole32(ICE_LO_DWORD(dma_buf->pa));
	}

	/* Debug desc and buffer */
	DNPRINTF(ICE_DBG_AQ_DESC, "ATQ: Control Send queue desc and buffer:\n");
	ice_debug_cq(hw, cq, (void *)desc_on_ring, buf, buf_size, false);

	(cq->sq.next_to_use)++;
	if (cq->sq.next_to_use == cq->sq.count)
		cq->sq.next_to_use = 0;
	ICE_WRITE(hw, cq->sq.tail, cq->sq.next_to_use);
	ice_flush(hw);

	/* Wait a short time before initial ice_sq_done() check, to allow
	 * hardware time for completion.
	 */
	ice_usec_delay(5, false);

	do {
		if (ice_sq_done(hw, cq))
			break;

		ice_usec_delay(10, false);
		total_delay++;
	} while (total_delay < cq->sq_cmd_timeout);

	/* if ready, copy the desc back to temp */
	if (ice_sq_done(hw, cq)) {
		memcpy(desc, desc_on_ring, sizeof(*desc));
		if (buf) {
			/* get returned length to copy */
			uint16_t copy_size = le16toh(desc->datalen);

			if (copy_size > buf_size) {
				DNPRINTF(ICE_DBG_AQ_MSG,
				    "Return len %d > than buf len %d\n",
				    copy_size, buf_size);
				status = ICE_ERR_AQ_ERROR;
			} else {
				memcpy(buf, dma_buf->va, copy_size);
			}
		}
		retval = le16toh(desc->retval);
		if (retval) {
			DNPRINTF(ICE_DBG_AQ_MSG, "Control Send Queue "
			    "command 0x%04X completed with error 0x%X\n",
			    le16toh(desc->opcode), retval);

			/* strip off FW internal code */
			retval &= 0xff;
		}
		cmd_completed = true;
		if (!status && retval != ICE_AQ_RC_OK)
			status = ICE_ERR_AQ_ERROR;
		cq->sq_last_status = (enum ice_aq_err)retval;
	}

	DNPRINTF(ICE_DBG_AQ_MSG, "ATQ: desc and buffer writeback:\n");
	ice_debug_cq(hw, cq, (void *)desc, buf, buf_size, true);

	/* save writeback AQ if requested */
	if (cd && cd->wb_desc)
		memcpy(cd->wb_desc, desc_on_ring, sizeof(*cd->wb_desc));

	/* update the error if time out occurred */
	if (!cmd_completed) {
		if (ICE_READ(hw, cq->rq.len) & cq->rq.len_crit_mask ||
		    ICE_READ(hw, cq->sq.len) & cq->sq.len_crit_mask) {
			DNPRINTF(ICE_DBG_AQ_MSG, "Critical FW error.\n");
			status = ICE_ERR_AQ_FW_CRITICAL;
		} else {
			DNPRINTF(ICE_DBG_AQ_MSG,
			    "Control Send Queue Writeback timeout.\n");
			status = ICE_ERR_AQ_TIMEOUT;
		}
	}

sq_send_command_error:
	return status;
}

/**
 * ice_sq_send_cmd - send command to a control queue
 * @hw: pointer to the HW struct
 * @cq: pointer to the specific Control queue
 * @desc: prefilled descriptor describing the command
 * @buf: buffer to use for indirect commands (or NULL for direct commands)
 * @buf_size: size of buffer for indirect commands (or 0 for direct commands)
 * @cd: pointer to command details structure
 *
 * Main command for the transmit side of a control queue. It puts the command
 * on the queue, bumps the tail, waits for processing of the command, captures
 * command status and results, etc.
 */
enum ice_status
ice_sq_send_cmd(struct ice_hw *hw, struct ice_ctl_q_info *cq,
		struct ice_aq_desc *desc, void *buf, uint16_t buf_size,
		struct ice_sq_cd *cd)
{
	enum ice_status status = ICE_SUCCESS;

	/* if reset is in progress return a soft error */
	if (hw->reset_ongoing)
		return ICE_ERR_RESET_ONGOING;
#if 0
	ice_acquire_lock(&cq->sq_lock);
#endif
	status = ice_sq_send_cmd_nolock(hw, cq, desc, buf, buf_size, cd);
#if 0
	ice_release_lock(&cq->sq_lock);
#endif
	return status;
}


/**
 * ice_sq_send_cmd_retry - send command to Control Queue (ATQ)
 * @hw: pointer to the HW struct
 * @cq: pointer to the specific Control queue
 * @desc: prefilled descriptor describing the command
 * @buf: buffer to use for indirect commands (or NULL for direct commands)
 * @buf_size: size of buffer for indirect commands (or 0 for direct commands)
 * @cd: pointer to command details structure
 *
 * Retry sending the FW Admin Queue command, multiple times, to the FW Admin
 * Queue if the EBUSY AQ error is returned.
 */
enum ice_status
ice_sq_send_cmd_retry(struct ice_hw *hw, struct ice_ctl_q_info *cq,
    struct ice_aq_desc *desc, void *buf, uint16_t buf_size,
    struct ice_sq_cd *cd)
{
	struct ice_aq_desc desc_cpy;
	enum ice_status status;
	bool is_cmd_for_retry;
	uint8_t *buf_cpy = NULL;
	uint8_t idx = 0;
	uint16_t opcode;

	opcode = le16toh(desc->opcode);
	is_cmd_for_retry = ice_should_retry_sq_send_cmd(opcode);
	memset(&desc_cpy, 0, sizeof(desc_cpy));

	if (is_cmd_for_retry) {
		if (buf) {
			buf_cpy = (uint8_t *)ice_malloc(hw, buf_size);
			if (!buf_cpy)
				return ICE_ERR_NO_MEMORY;
		}

		memcpy(&desc_cpy, desc, sizeof(desc_cpy));
	}

	do {
		status = ice_sq_send_cmd(hw, cq, desc, buf, buf_size, cd);

		if (!is_cmd_for_retry || status == ICE_SUCCESS ||
		    hw->adminq.sq_last_status != ICE_AQ_RC_EBUSY)
			break;

		if (buf_cpy)
			memcpy(buf, buf_cpy, buf_size);

		memcpy(desc, &desc_cpy, sizeof(desc_cpy));

		ice_msec_delay(ICE_SQ_SEND_DELAY_TIME_MS, false);

	} while (++idx < ICE_SQ_SEND_MAX_EXECUTE);

	if (buf_cpy)
		ice_free(hw, buf_cpy);

	return status;
}

/**
 * ice_aq_send_cmd - send FW Admin Queue command to FW Admin Queue
 * @hw: pointer to the HW struct
 * @desc: descriptor describing the command
 * @buf: buffer to use for indirect commands (NULL for direct commands)
 * @buf_size: size of buffer for indirect commands (0 for direct commands)
 * @cd: pointer to command details structure
 *
 * Helper function to send FW Admin Queue commands to the FW Admin Queue.
 */
enum ice_status
ice_aq_send_cmd(struct ice_hw *hw, struct ice_aq_desc *desc, void *buf,
		uint16_t buf_size, struct ice_sq_cd *cd)
{
	return ice_sq_send_cmd_retry(hw, &hw->adminq, desc, buf, buf_size, cd);
}

/**
 * ice_write_byte - write a byte to a packed context structure
 * @src_ctx:  the context structure to read from
 * @dest_ctx: the context to be written to
 * @ce_info:  a description of the struct to be filled
 */
void
ice_write_byte(uint8_t *src_ctx, uint8_t *dest_ctx,
    const struct ice_ctx_ele *ce_info)
{
	uint8_t src_byte, dest_byte, mask;
	uint8_t *from, *dest;
	uint16_t shift_width;

	/* copy from the next struct field */
	from = src_ctx + ce_info->offset;

	/* prepare the bits and mask */
	shift_width = ce_info->lsb % 8;
	mask = (uint8_t)(BIT(ce_info->width) - 1);

	src_byte = *from;
	src_byte &= mask;

	/* shift to correct alignment */
	mask <<= shift_width;
	src_byte <<= shift_width;

	/* get the current bits from the target bit string */
	dest = dest_ctx + (ce_info->lsb / 8);

	memcpy(&dest_byte, dest, sizeof(dest_byte));

	dest_byte &= ~mask;	/* get the bits not changing */
	dest_byte |= src_byte;	/* add in the new bits */

	/* put it all back */
	memcpy(dest, &dest_byte, sizeof(dest_byte));
}

/**
 * ice_write_word - write a word to a packed context structure
 * @src_ctx:  the context structure to read from
 * @dest_ctx: the context to be written to
 * @ce_info:  a description of the struct to be filled
 */
void
ice_write_word(uint8_t *src_ctx, uint8_t *dest_ctx,
    const struct ice_ctx_ele *ce_info)
{
	uint16_t src_word, mask;
	uint16_t dest_word;
	uint8_t *from, *dest;
	uint16_t shift_width;

	/* copy from the next struct field */
	from = src_ctx + ce_info->offset;

	/* prepare the bits and mask */
	shift_width = ce_info->lsb % 8;
	mask = BIT(ce_info->width) - 1;

	/* don't swizzle the bits until after the mask because the mask bits
	 * will be in a different bit position on big endian machines
	 */
	src_word = *(uint16_t *)from;
	src_word &= mask;

	/* shift to correct alignment */
	mask <<= shift_width;
	src_word <<= shift_width;

	/* get the current bits from the target bit string */
	dest = dest_ctx + (ce_info->lsb / 8);

	memcpy(&dest_word, dest, sizeof(dest_word));

	dest_word &= ~(htole16(mask));	/* get the bits not changing */
	dest_word |= htole16(src_word);	/* add in the new bits */

	/* put it all back */
	memcpy(dest, &dest_word, sizeof(dest_word));
}

/**
 * ice_write_dword - write a dword to a packed context structure
 * @src_ctx:  the context structure to read from
 * @dest_ctx: the context to be written to
 * @ce_info:  a description of the struct to be filled
 */
void
ice_write_dword(uint8_t *src_ctx, uint8_t *dest_ctx,
    const struct ice_ctx_ele *ce_info)
{
	uint32_t src_dword, mask;
	uint32_t dest_dword;
	uint8_t *from, *dest;
	uint16_t shift_width;

	/* copy from the next struct field */
	from = src_ctx + ce_info->offset;

	/* prepare the bits and mask */
	shift_width = ce_info->lsb % 8;

	/* if the field width is exactly 32 on an x86 machine, then the shift
	 * operation will not work because the SHL instructions count is masked
	 * to 5 bits so the shift will do nothing
	 */
	if (ce_info->width < 32)
		mask = BIT(ce_info->width) - 1;
	else
		mask = (uint32_t)~0;

	/* don't swizzle the bits until after the mask because the mask bits
	 * will be in a different bit position on big endian machines
	 */
	src_dword = *(uint32_t *)from;
	src_dword &= mask;

	/* shift to correct alignment */
	mask <<= shift_width;
	src_dword <<= shift_width;

	/* get the current bits from the target bit string */
	dest = dest_ctx + (ce_info->lsb / 8);

	memcpy(&dest_dword, dest, sizeof(dest_dword));

	dest_dword &= ~(htole32(mask));	/* get the bits not changing */
	dest_dword |= htole32(src_dword);	/* add in the new bits */

	/* put it all back */
	memcpy(dest, &dest_dword, sizeof(dest_dword));
}

/**
 * ice_write_qword - write a qword to a packed context structure
 * @src_ctx:  the context structure to read from
 * @dest_ctx: the context to be written to
 * @ce_info:  a description of the struct to be filled
 */
void
ice_write_qword(uint8_t *src_ctx, uint8_t *dest_ctx,
    const struct ice_ctx_ele *ce_info)
{
	uint64_t src_qword, mask;
	uint64_t dest_qword;
	uint8_t *from, *dest;
	uint16_t shift_width;

	/* copy from the next struct field */
	from = src_ctx + ce_info->offset;

	/* prepare the bits and mask */
	shift_width = ce_info->lsb % 8;

	/* if the field width is exactly 64 on an x86 machine, then the shift
	 * operation will not work because the SHL instructions count is masked
	 * to 6 bits so the shift will do nothing
	 */
	if (ce_info->width < 64)
		mask = BIT_ULL(ce_info->width) - 1;
	else
		mask = (uint64_t)~0;

	/* don't swizzle the bits until after the mask because the mask bits
	 * will be in a different bit position on big endian machines
	 */
	src_qword = *(uint64_t *)from;
	src_qword &= mask;

	/* shift to correct alignment */
	mask <<= shift_width;
	src_qword <<= shift_width;

	/* get the current bits from the target bit string */
	dest = dest_ctx + (ce_info->lsb / 8);

	memcpy(&dest_qword, dest, sizeof(dest_qword) );

	dest_qword &= ~(htole64(mask));	/* get the bits not changing */
	dest_qword |= htole64(src_qword);	/* add in the new bits */

	/* put it all back */
	memcpy(dest, &dest_qword, sizeof(dest_qword));
}

/**
 * ice_set_ctx - set context bits in packed structure
 * @hw: pointer to the hardware structure
 * @src_ctx:  pointer to a generic non-packed context structure
 * @dest_ctx: pointer to memory for the packed structure
 * @ce_info:  a description of the structure to be transformed
 */
enum ice_status
ice_set_ctx(struct ice_hw *hw, uint8_t *src_ctx, uint8_t *dest_ctx,
    const struct ice_ctx_ele *ce_info)
{
	int f;

	for (f = 0; ce_info[f].width; f++) {
		/* We have to deal with each element of the FW response
		 * using the correct size so that we are correct regardless
		 * of the endianness of the machine.
		 */
		if (ce_info[f].width > (ce_info[f].size_of * 8)) {
			DNPRINTF(ICE_DBG_QCTX, "%s: Field %d width of %d bits "
			    "larger than size of %d byte(s); skipping write\n",
			    __func__, f, ce_info[f].width, ce_info[f].size_of);
			continue;
		}
		switch (ce_info[f].size_of) {
		case sizeof(uint8_t):
			ice_write_byte(src_ctx, dest_ctx, &ce_info[f]);
			break;
		case sizeof(uint16_t):
			ice_write_word(src_ctx, dest_ctx, &ce_info[f]);
			break;
		case sizeof(uint32_t):
			ice_write_dword(src_ctx, dest_ctx, &ce_info[f]);
			break;
		case sizeof(uint64_t):
			ice_write_qword(src_ctx, dest_ctx, &ce_info[f]);
			break;
		default:
			return ICE_ERR_INVAL_SIZE;
		}
	}

	return ICE_SUCCESS;
}

/**
 * ice_fill_dflt_direct_cmd_desc - AQ descriptor helper function
 * @desc: pointer to the temp descriptor (non DMA mem)
 * @opcode: the opcode can be used to decide which flags to turn off or on
 *
 * Fill the desc with default values
 */
void
ice_fill_dflt_direct_cmd_desc(struct ice_aq_desc *desc, uint16_t opcode)
{
	/* zero out the desc */
	memset(desc, 0, sizeof(*desc));
	desc->opcode = htole16(opcode);
	desc->flags = htole16(ICE_AQ_FLAG_SI);
}

/**
 * ice_aq_get_fw_ver
 * @hw: pointer to the HW struct
 * @cd: pointer to command details structure or NULL
 *
 * Get the firmware version (0x0001) from the admin queue commands
 */
enum ice_status
ice_aq_get_fw_ver(struct ice_hw *hw, struct ice_sq_cd *cd)
{
	struct ice_aqc_get_ver *resp;
	struct ice_aq_desc desc;
	enum ice_status status;

	resp = &desc.params.get_ver;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_get_ver);

	status = ice_aq_send_cmd(hw, &desc, NULL, 0, cd);

	if (!status) {
		hw->fw_branch = resp->fw_branch;
		hw->fw_maj_ver = resp->fw_major;
		hw->fw_min_ver = resp->fw_minor;
		hw->fw_patch = resp->fw_patch;
		hw->fw_build = le32toh(resp->fw_build);
		hw->api_branch = resp->api_branch;
		hw->api_maj_ver = resp->api_major;
		hw->api_min_ver = resp->api_minor;
		hw->api_patch = resp->api_patch;
	}

	return status;
}

/*
 * ice_aq_q_shutdown
 * @hw: pointer to the HW struct
 * @unloading: is the driver unloading itself
 *
 * Tell the Firmware that we're shutting down the AdminQ and whether
 * or not the driver is unloading as well (0x0003).
 */
enum ice_status
ice_aq_q_shutdown(struct ice_hw *hw, bool unloading)
{
	struct ice_aqc_q_shutdown *cmd;
	struct ice_aq_desc desc;

	cmd = &desc.params.q_shutdown;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_q_shutdown);

	if (unloading)
		cmd->driver_unloading = ICE_AQC_DRIVER_UNLOADING;

	return ice_aq_send_cmd(hw, &desc, NULL, 0, NULL);
}

/**
 * ice_aq_req_res
 * @hw: pointer to the HW struct
 * @res: resource ID
 * @access: access type
 * @sdp_number: resource number
 * @timeout: the maximum time in ms that the driver may hold the resource
 * @cd: pointer to command details structure or NULL
 *
 * Requests common resource using the admin queue commands (0x0008).
 * When attempting to acquire the Global Config Lock, the driver can
 * learn of three states:
 *  1) ICE_SUCCESS -        acquired lock, and can perform download package
 *  2) ICE_ERR_AQ_ERROR -   did not get lock, driver should fail to load
 *  3) ICE_ERR_AQ_NO_WORK - did not get lock, but another driver has
 *                          successfully downloaded the package; the driver does
 *                          not have to download the package and can continue
 *                          loading
 *
 * Note that if the caller is in an acquire lock, perform action, release lock
 * phase of operation, it is possible that the FW may detect a timeout and issue
 * a CORER. In this case, the driver will receive a CORER interrupt and will
 * have to determine its cause. The calling thread that is handling this flow
 * will likely get an error propagated back to it indicating the Download
 * Package, Update Package or the Release Resource AQ commands timed out.
 */
enum ice_status
ice_aq_req_res(struct ice_hw *hw, enum ice_aq_res_ids res,
	       enum ice_aq_res_access_type access, uint8_t sdp_number,
	       uint32_t *timeout, struct ice_sq_cd *cd)
{
	struct ice_aqc_req_res *cmd_resp;
	struct ice_aq_desc desc;
	enum ice_status status;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);

	cmd_resp = &desc.params.res_owner;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_req_res);

	cmd_resp->res_id = htole16(res);
	cmd_resp->access_type = htole16(access);
	cmd_resp->res_number = htole32(sdp_number);
	cmd_resp->timeout = htole32(*timeout);
	*timeout = 0;

	status = ice_aq_send_cmd(hw, &desc, NULL, 0, cd);

	/* The completion specifies the maximum time in ms that the driver
	 * may hold the resource in the Timeout field.
	 */

	/* Global config lock response utilizes an additional status field.
	 *
	 * If the Global config lock resource is held by some other driver, the
	 * command completes with ICE_AQ_RES_GLBL_IN_PROG in the status field
	 * and the timeout field indicates the maximum time the current owner
	 * of the resource has to free it.
	 */
	if (res == ICE_GLOBAL_CFG_LOCK_RES_ID) {
		if (le16toh(cmd_resp->status) == ICE_AQ_RES_GLBL_SUCCESS) {
			*timeout = le32toh(cmd_resp->timeout);
			return ICE_SUCCESS;
		} else if (le16toh(cmd_resp->status) ==
			   ICE_AQ_RES_GLBL_IN_PROG) {
			*timeout = le32toh(cmd_resp->timeout);
			return ICE_ERR_AQ_ERROR;
		} else if (le16toh(cmd_resp->status) ==
			   ICE_AQ_RES_GLBL_DONE) {
			return ICE_ERR_AQ_NO_WORK;
		}

		/* invalid FW response, force a timeout immediately */
		*timeout = 0;
		return ICE_ERR_AQ_ERROR;
	}

	/* If the resource is held by some other driver, the command completes
	 * with a busy return value and the timeout field indicates the maximum
	 * time the current owner of the resource has to free it.
	 */
	if (!status || hw->adminq.sq_last_status == ICE_AQ_RC_EBUSY)
		*timeout = le32toh(cmd_resp->timeout);

	return status;
}

/**
 * ice_aq_release_res
 * @hw: pointer to the HW struct
 * @res: resource ID
 * @sdp_number: resource number
 * @cd: pointer to command details structure or NULL
 *
 * release common resource using the admin queue commands (0x0009)
 */
enum ice_status
ice_aq_release_res(struct ice_hw *hw, enum ice_aq_res_ids res,
    uint8_t sdp_number, struct ice_sq_cd *cd)
{
	struct ice_aqc_req_res *cmd;
	struct ice_aq_desc desc;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);

	cmd = &desc.params.res_owner;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_release_res);

	cmd->res_id = htole16(res);
	cmd->res_number = htole32(sdp_number);

	return ice_aq_send_cmd(hw, &desc, NULL, 0, cd);
}

/**
 * ice_acquire_res
 * @hw: pointer to the HW structure
 * @res: resource ID
 * @access: access type (read or write)
 * @timeout: timeout in milliseconds
 *
 * This function will attempt to acquire the ownership of a resource.
 */
enum ice_status
ice_acquire_res(struct ice_hw *hw, enum ice_aq_res_ids res,
    enum ice_aq_res_access_type access, uint32_t timeout)
{
#define ICE_RES_POLLING_DELAY_MS	10
	uint32_t delay = ICE_RES_POLLING_DELAY_MS;
	uint32_t time_left = timeout;
	enum ice_status status;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);

	status = ice_aq_req_res(hw, res, access, 0, &time_left, NULL);

	/* A return code of ICE_ERR_AQ_NO_WORK means that another driver has
	 * previously acquired the resource and performed any necessary updates;
	 * in this case the caller does not obtain the resource and has no
	 * further work to do.
	 */
	if (status == ICE_ERR_AQ_NO_WORK)
		goto ice_acquire_res_exit;

	if (status)
		DNPRINTF(ICE_DBG_RES, "resource %d acquire type %d failed.\n",
		    res, access);

	/* If necessary, poll until the current lock owner timeouts */
	timeout = time_left;
	while (status && timeout && time_left) {
		ice_msec_delay(delay, true);
		timeout = (timeout > delay) ? timeout - delay : 0;
		status = ice_aq_req_res(hw, res, access, 0, &time_left, NULL);

		if (status == ICE_ERR_AQ_NO_WORK)
			/* lock free, but no work to do */
			break;

		if (!status)
			/* lock acquired */
			break;
	}
	if (status && status != ICE_ERR_AQ_NO_WORK)
		DNPRINTF(ICE_DBG_RES, "resource acquire timed out.\n");

ice_acquire_res_exit:
	if (status == ICE_ERR_AQ_NO_WORK) {
		if (access == ICE_RES_WRITE)
			DNPRINTF(ICE_DBG_RES,
			    "resource indicates no work to do.\n");
		else
			DNPRINTF(ICE_DBG_RES,
			    "Warning: ICE_ERR_AQ_NO_WORK not expected\n");
	}
	return status;
}

/**
 * ice_release_res
 * @hw: pointer to the HW structure
 * @res: resource ID
 *
 * This function will release a resource using the proper Admin Command.
 */
void ice_release_res(struct ice_hw *hw, enum ice_aq_res_ids res)
{
	enum ice_status status;
	uint32_t total_delay = 0;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);

	status = ice_aq_release_res(hw, res, 0, NULL);

	/* there are some rare cases when trying to release the resource
	 * results in an admin queue timeout, so handle them correctly
	 */
	while ((status == ICE_ERR_AQ_TIMEOUT) &&
	       (total_delay < hw->adminq.sq_cmd_timeout)) {
		ice_msec_delay(1, true);
		status = ice_aq_release_res(hw, res, 0, NULL);
		total_delay++;
	}
}

/*
 * ice_aq_ver_check - Check the reported AQ API version
 * Checks if the driver should load on a given AQ API version.
 * Return: 'true' iff the driver should attempt to load. 'false' otherwise.
 */
bool
ice_aq_ver_check(struct ice_hw *hw)
{
	struct ice_softc *sc = hw->hw_sc;

	if (hw->api_maj_ver > EXP_FW_API_VER_MAJOR) {
		/* Major API version is newer than expected, don't load */
		printf("%s: unsupported firmware API major version %u; "
		    "expected version is %u\n", sc->sc_dev.dv_xname,
		    hw->api_maj_ver, EXP_FW_API_VER_MAJOR);
		return false;
	}

	return true;
}

/*
 * ice_shutdown_sq - shutdown the transmit side of a control queue
 * @hw: pointer to the hardware structure
 * @cq: pointer to the specific Control queue
 *
 * The main shutdown routine for the Control Transmit Queue
 */
enum ice_status
ice_shutdown_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
{
	enum ice_status ret_code = ICE_SUCCESS;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);
#if 0
	ice_acquire_lock(&cq->sq_lock);
#endif
	if (!cq->sq.count) {
		ret_code = ICE_ERR_NOT_READY;
		goto shutdown_sq_out;
	}

	/* Stop processing of the control queue */
	ICE_WRITE(hw, cq->sq.head, 0);
	ICE_WRITE(hw, cq->sq.tail, 0);
	ICE_WRITE(hw, cq->sq.len, 0);
	ICE_WRITE(hw, cq->sq.bal, 0);
	ICE_WRITE(hw, cq->sq.bah, 0);

	cq->sq.count = 0;	/* to indicate uninitialized queue */

	/* free ring buffers and the ring itself */
	ICE_FREE_CQ_BUFS(hw, cq, sq);
	ice_free_cq_ring(hw, &cq->sq);

shutdown_sq_out:
#if 0
	ice_release_lock(&cq->sq_lock);
#endif
	return ret_code;
}

/*
 * ice_shutdown_rq - shutdown Control ARQ
 * @hw: pointer to the hardware structure
 * @cq: pointer to the specific Control queue
 *
 * The main shutdown routine for the Control Receive Queue
 */
enum ice_status
ice_shutdown_rq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
{
	enum ice_status ret_code = ICE_SUCCESS;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);
#if 0
	ice_acquire_lock(&cq->rq_lock);
#endif
	if (!cq->rq.count) {
		ret_code = ICE_ERR_NOT_READY;
		goto shutdown_rq_out;
	}

	/* Stop Control Queue processing */
	ICE_WRITE(hw, cq->rq.head, 0);
	ICE_WRITE(hw, cq->rq.tail, 0);
	ICE_WRITE(hw, cq->rq.len, 0);
	ICE_WRITE(hw, cq->rq.bal, 0);
	ICE_WRITE(hw, cq->rq.bah, 0);

	/* set rq.count to 0 to indicate uninitialized queue */
	cq->rq.count = 0;

	/* free ring buffers and the ring itself */
	ICE_FREE_CQ_BUFS(hw, cq, rq);
	ice_free_cq_ring(hw, &cq->rq);

shutdown_rq_out:
#if 0
	ice_release_lock(&cq->rq_lock);
#endif
	return ret_code;
}

/*
 * ice_init_check_adminq - Check version for Admin Queue to know if its alive
 */
enum ice_status
ice_init_check_adminq(struct ice_hw *hw)
{
	struct ice_ctl_q_info *cq = &hw->adminq;
	enum ice_status status;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);

	status = ice_aq_get_fw_ver(hw, NULL);
	if (status)
		goto init_ctrlq_free_rq;

	if (!ice_aq_ver_check(hw)) {
		status = ICE_ERR_FW_API_VER;
		goto init_ctrlq_free_rq;
	}

	return ICE_SUCCESS;

init_ctrlq_free_rq:
	ice_shutdown_rq(hw, cq);
	ice_shutdown_sq(hw, cq);
	return status;
}

/*
 * ice_init_ctrlq - main initialization routine for any control Queue
 * @hw: pointer to the hardware structure
 * @q_type: specific Control queue type
 *
 * Prior to calling this function, the driver *MUST* set the following fields
 * in the cq->structure:
 *     - cq->num_sq_entries
 *     - cq->num_rq_entries
 *     - cq->rq_buf_size
 *     - cq->sq_buf_size
 *
 * NOTE: this function does not initialize the controlq locks
 */
enum ice_status
ice_init_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type)
{
	struct ice_ctl_q_info *cq;
	enum ice_status ret_code;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);

	switch (q_type) {
	case ICE_CTL_Q_ADMIN:
		ice_adminq_init_regs(hw);
		cq = &hw->adminq;
		break;
	case ICE_CTL_Q_MAILBOX:
		ice_mailbox_init_regs(hw);
		cq = &hw->mailboxq;
		break;
	default:
		return ICE_ERR_PARAM;
	}
	cq->qtype = q_type;

	/* verify input for valid configuration */
	if (!cq->num_rq_entries || !cq->num_sq_entries ||
	    !cq->rq_buf_size || !cq->sq_buf_size) {
		return ICE_ERR_CFG;
	}

	/* setup SQ command write back timeout */
	cq->sq_cmd_timeout = ICE_CTL_Q_SQ_CMD_TIMEOUT;

	/* allocate the ATQ */
	ret_code = ice_init_sq(hw, cq);
	if (ret_code)
		return ret_code;

	/* allocate the ARQ */
	ret_code = ice_init_rq(hw, cq);
	if (ret_code)
		goto init_ctrlq_free_sq;

	/* success! */
	return ICE_SUCCESS;

init_ctrlq_free_sq:
	ice_shutdown_sq(hw, cq);
	return ret_code;
}

/*
 * ice_shutdown_ctrlq - shutdown routine for any control queue
 * @hw: pointer to the hardware structure
 * @q_type: specific Control queue type
 * @unloading: is the driver unloading itself
 *
 * NOTE: this function does not destroy the control queue locks.
 */
void
ice_shutdown_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type,
		   bool unloading)
{
	struct ice_ctl_q_info *cq;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);

	switch (q_type) {
	case ICE_CTL_Q_ADMIN:
		cq = &hw->adminq;
		if (ice_check_sq_alive(hw, cq))
			ice_aq_q_shutdown(hw, unloading);
		break;
	case ICE_CTL_Q_MAILBOX:
		cq = &hw->mailboxq;
		break;
	default:
		return;
	}

	ice_shutdown_sq(hw, cq);
	ice_shutdown_rq(hw, cq);
}

/*
 * ice_shutdown_all_ctrlq - shutdown routine for all control queues
 * @hw: pointer to the hardware structure
 * @unloading: is the driver unloading itself
 *
 * NOTE: this function does not destroy the control queue locks. The driver
 * may call this at runtime to shutdown and later restart control queues, such
 * as in response to a reset event.
 */
void
ice_shutdown_all_ctrlq(struct ice_hw *hw, bool unloading)
{
	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);
	/* Shutdown FW admin queue */
	ice_shutdown_ctrlq(hw, ICE_CTL_Q_ADMIN, unloading);
	/* Shutdown PF-VF Mailbox */
	ice_shutdown_ctrlq(hw, ICE_CTL_Q_MAILBOX, unloading);
}

/**
 * ice_init_all_ctrlq - main initialization routine for all control queues
 * @hw: pointer to the hardware structure
 *
 * Prior to calling this function, the driver MUST* set the following fields
 * in the cq->structure for all control queues:
 *     - cq->num_sq_entries
 *     - cq->num_rq_entries
 *     - cq->rq_buf_size
 *     - cq->sq_buf_size
 *
 * NOTE: this function does not initialize the controlq locks.
 */
enum ice_status
ice_init_all_ctrlq(struct ice_hw *hw)
{
	enum ice_status status;
	uint32_t retry = 0;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);

	/* Init FW admin queue */
	do {
		status = ice_init_ctrlq(hw, ICE_CTL_Q_ADMIN);
		if (status)
			return status;

		status = ice_init_check_adminq(hw);
		if (status != ICE_ERR_AQ_FW_CRITICAL)
			break;

		DNPRINTF(ICE_DBG_AQ_MSG, "Retry Admin Queue init due to FW critical error\n");
		ice_shutdown_ctrlq(hw, ICE_CTL_Q_ADMIN, true);
		ice_msec_delay(ICE_CTL_Q_ADMIN_INIT_MSEC, true);
	} while (retry++ < ICE_CTL_Q_ADMIN_INIT_TIMEOUT);

	if (status)
		return status;

	/* Init Mailbox queue */
	return ice_init_ctrlq(hw, ICE_CTL_Q_MAILBOX);
}

/**
 * ice_init_ctrlq_locks - Initialize locks for a control queue
 * @cq: pointer to the control queue
 *
 * Initializes the send and receive queue locks for a given control queue.
 */
void
ice_init_ctrlq_locks(struct ice_ctl_q_info *cq)
{
	ice_init_lock(&cq->sq_lock);
	ice_init_lock(&cq->rq_lock);
}

/**
 * ice_create_all_ctrlq - main initialization routine for all control queues
 * @hw: pointer to the hardware structure
 *
 * Prior to calling this function, the driver *MUST* set the following fields
 * in the cq->structure for all control queues:
 *     - cq->num_sq_entries
 *     - cq->num_rq_entries
 *     - cq->rq_buf_size
 *     - cq->sq_buf_size
 *
 * This function creates all the control queue locks and then calls
 * ice_init_all_ctrlq. It should be called once during driver load. If the
 * driver needs to re-initialize control queues at run time it should call
 * ice_init_all_ctrlq instead.
 */
enum ice_status
ice_create_all_ctrlq(struct ice_hw *hw)
{
	ice_init_ctrlq_locks(&hw->adminq);
	ice_init_ctrlq_locks(&hw->mailboxq);

	return ice_init_all_ctrlq(hw);
}

/*
 * ice_destroy_all_ctrlq - exit routine for all control queues
 *
 * This function shuts down all the control queues and then destroys the
 * control queue locks. It should be called once during driver unload. The
 * driver should call ice_shutdown_all_ctrlq if it needs to shut down and
 * reinitialize control queues, such as in response to a reset event.
 */
void
ice_destroy_all_ctrlq(struct ice_hw *hw)
{
	/* shut down all the control queues first */
	ice_shutdown_all_ctrlq(hw, true);

	ice_destroy_ctrlq_locks(&hw->adminq);
	ice_destroy_ctrlq_locks(&hw->mailboxq);
}

/**
 * ice_aq_fwlog_get - Get the current firmware logging configuration (0xFF32)
 * @hw: pointer to the HW structure
 * @cfg: firmware logging configuration to populate
 */
enum ice_status
ice_aq_fwlog_get(struct ice_hw *hw, struct ice_fwlog_cfg *cfg)
{
	struct ice_aqc_fw_log_cfg_resp *fw_modules;
	struct ice_aqc_fw_log *cmd;
	struct ice_aq_desc desc;
	enum ice_status status;
	uint16_t i, module_id_cnt;
	void *buf;

	memset(cfg, 0, sizeof(*cfg));

	buf = ice_calloc(hw, 1, ICE_AQ_MAX_BUF_LEN);
	if (!buf)
		return ICE_ERR_NO_MEMORY;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_fw_logs_query);
	cmd = &desc.params.fw_log;

	cmd->cmd_flags = ICE_AQC_FW_LOG_AQ_QUERY;

	status = ice_aq_send_cmd(hw, &desc, buf, ICE_AQ_MAX_BUF_LEN, NULL);
	if (status) {
		DNPRINTF(ICE_DBG_FW_LOG,
		    "Failed to get FW log configuration\n");
		goto status_out;
	}

	module_id_cnt = le16toh(cmd->ops.cfg.mdl_cnt);
	if (module_id_cnt < ICE_AQC_FW_LOG_ID_MAX) {
		DNPRINTF(ICE_DBG_FW_LOG, "FW returned less than the expected "
		    "number of FW log module IDs\n");
	} else {
		if (module_id_cnt > ICE_AQC_FW_LOG_ID_MAX)
			DNPRINTF(ICE_DBG_FW_LOG, "FW returned more than "
			    "expected number of FW log module IDs, setting "
			    "module_id_cnt to software expected max %u\n",
			    ICE_AQC_FW_LOG_ID_MAX);
		module_id_cnt = ICE_AQC_FW_LOG_ID_MAX;
	}

	cfg->log_resolution = le16toh(cmd->ops.cfg.log_resolution);
	if (cmd->cmd_flags & ICE_AQC_FW_LOG_CONF_AQ_EN)
		cfg->options |= ICE_FWLOG_OPTION_ARQ_ENA;
	if (cmd->cmd_flags & ICE_AQC_FW_LOG_CONF_UART_EN)
		cfg->options |= ICE_FWLOG_OPTION_UART_ENA;
	if (cmd->cmd_flags & ICE_AQC_FW_LOG_QUERY_REGISTERED)
		cfg->options |= ICE_FWLOG_OPTION_IS_REGISTERED;

	fw_modules = (struct ice_aqc_fw_log_cfg_resp *)buf;

	for (i = 0; i < module_id_cnt; i++) {
		struct ice_aqc_fw_log_cfg_resp *fw_module = &fw_modules[i];

		cfg->module_entries[i].module_id =
			le16toh(fw_module->module_identifier);
		cfg->module_entries[i].log_level = fw_module->log_level;
	}

status_out:
	ice_free(hw, buf);
	return status;
}

/**
 * ice_fwlog_set_support_ena - Set if FW logging is supported by FW
 * @hw: pointer to the HW struct
 *
 * If FW returns success to the ice_aq_fwlog_get call then it supports FW
 * logging, else it doesn't. Set the fwlog_support_ena flag accordingly.
 *
 * This function is only meant to be called during driver init to determine if
 * the FW support FW logging.
 */
void
ice_fwlog_set_support_ena(struct ice_hw *hw)
{
	struct ice_fwlog_cfg *cfg;
	enum ice_status status;

	hw->fwlog_support_ena = false;

	cfg = (struct ice_fwlog_cfg *)ice_calloc(hw, 1, sizeof(*cfg));
	if (!cfg)
		return;

	/* don't call ice_fwlog_get() because that would overwrite the cached
	 * configuration from the call to ice_fwlog_init(), which is expected to
	 * be called prior to this function
	 */
	status = ice_aq_fwlog_get(hw, cfg);
	if (status)
		DNPRINTF(ICE_DBG_FW_LOG, "ice_fwlog_get failed, FW logging "
		    "is not supported on this version of FW, status %d\n",
		    status);
	else
		hw->fwlog_support_ena = true;

	ice_free(hw, cfg);
}

/**
 * ice_aq_fwlog_set - Set FW logging configuration AQ command (0xFF30)
 * @hw: pointer to the HW structure
 * @entries: entries to configure
 * @num_entries: number of @entries
 * @options: options from ice_fwlog_cfg->options structure
 * @log_resolution: logging resolution
 */
enum ice_status
ice_aq_fwlog_set(struct ice_hw *hw, struct ice_fwlog_module_entry *entries,
		 uint16_t num_entries, uint16_t options,
		 uint16_t log_resolution)
{
	struct ice_aqc_fw_log_cfg_resp *fw_modules;
	struct ice_aqc_fw_log *cmd;
	struct ice_aq_desc desc;
	enum ice_status status;
	uint16_t i;

	fw_modules = (struct ice_aqc_fw_log_cfg_resp *)
		ice_calloc(hw, num_entries, sizeof(*fw_modules));
	if (!fw_modules)
		return ICE_ERR_NO_MEMORY;

	for (i = 0; i < num_entries; i++) {
		fw_modules[i].module_identifier = htole16(entries[i].module_id);
		fw_modules[i].log_level = entries[i].log_level;
	}

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_fw_logs_config);
	desc.flags |= htole16(ICE_AQ_FLAG_RD);

	cmd = &desc.params.fw_log;

	cmd->cmd_flags = ICE_AQC_FW_LOG_CONF_SET_VALID;
	cmd->ops.cfg.log_resolution = htole16(log_resolution);
	cmd->ops.cfg.mdl_cnt = htole16(num_entries);

	if (options & ICE_FWLOG_OPTION_ARQ_ENA)
		cmd->cmd_flags |= ICE_AQC_FW_LOG_CONF_AQ_EN;
	if (options & ICE_FWLOG_OPTION_UART_ENA)
		cmd->cmd_flags |= ICE_AQC_FW_LOG_CONF_UART_EN;

	status = ice_aq_send_cmd(hw, &desc, fw_modules,
				 sizeof(*fw_modules) * num_entries,
				 NULL);

	ice_free(hw, fw_modules);

	return status;
}

/**
 * ice_fwlog_supported - Cached for whether FW supports FW logging or not
 * @hw: pointer to the HW structure
 *
 * This will always return false if called before ice_init_hw(), so it must be
 * called after ice_init_hw().
 */
bool
ice_fwlog_supported(struct ice_hw *hw)
{
	return hw->fwlog_support_ena;
}

/**
 * ice_fwlog_valid_module_entries - validate all the module entry IDs and
 * log levels
 */
bool
ice_fwlog_valid_module_entries(struct ice_hw *hw,
    struct ice_fwlog_module_entry *entries, uint16_t num_entries)
{
	uint16_t i;

	if (!entries) {
		DNPRINTF(ICE_DBG_FW_LOG, "Null ice_fwlog_module_entry array\n");
		return false;
	}

	if (!num_entries) {
		DNPRINTF(ICE_DBG_FW_LOG, "num_entries must be non-zero\n");
		return false;
	}

	for (i = 0; i < num_entries; i++) {
		struct ice_fwlog_module_entry *entry = &entries[i];

		if (entry->module_id >= ICE_AQC_FW_LOG_ID_MAX) {
			DNPRINTF(ICE_DBG_FW_LOG,
			    "Invalid module_id %u, max valid module_id is %u\n",
			    entry->module_id, ICE_AQC_FW_LOG_ID_MAX - 1);
			return false;
		}

		if (entry->log_level >= ICE_FWLOG_LEVEL_INVALID) {
			DNPRINTF(ICE_DBG_FW_LOG,
			    "Invalid log_level %u, max valid log_level is %u\n",
			    entry->log_level, ICE_AQC_FW_LOG_ID_MAX - 1);
			return false;
		}
	}

	return true;
}

/**
 * ice_fwlog_valid_cfg - validate entire configuration
 * @hw: pointer to the HW structure
 * @cfg: config to validate
 */
bool
ice_fwlog_valid_cfg(struct ice_hw *hw, struct ice_fwlog_cfg *cfg)
{
	if (!cfg) {
		DNPRINTF(ICE_DBG_FW_LOG, "Null ice_fwlog_cfg\n");
		return false;
	}

	if (cfg->log_resolution < ICE_AQC_FW_LOG_MIN_RESOLUTION ||
	    cfg->log_resolution > ICE_AQC_FW_LOG_MAX_RESOLUTION) {
		DNPRINTF(ICE_DBG_FW_LOG, "Unsupported log_resolution %u, "
		    "must be between %u and %u\n",
		    cfg->log_resolution, ICE_AQC_FW_LOG_MIN_RESOLUTION,
		    ICE_AQC_FW_LOG_MAX_RESOLUTION);
		return false;
	}

	if (!ice_fwlog_valid_module_entries(hw, cfg->module_entries,
				  ICE_AQC_FW_LOG_ID_MAX))
		return false;

	return true;
}

/**
 * ice_fwlog_set - Set the firmware logging settings
 * @hw: pointer to the HW structure
 * @cfg: config used to set firmware logging
 *
 * This function should be called whenever the driver needs to set the firmware
 * logging configuration. It can be called on initialization, reset, or during
 * runtime.
 *
 * If the PF wishes to receive FW logging then it must register via
 * ice_fwlog_register. Note, that ice_fwlog_register does not need to be called
 * for init.
 */
enum ice_status
ice_fwlog_set(struct ice_hw *hw, struct ice_fwlog_cfg *cfg)
{
	enum ice_status status;

	if (!ice_fwlog_supported(hw))
		return ICE_ERR_NOT_SUPPORTED;

	if (!ice_fwlog_valid_cfg(hw, cfg))
		return ICE_ERR_PARAM;

	status = ice_aq_fwlog_set(hw, cfg->module_entries,
				  ICE_AQC_FW_LOG_ID_MAX, cfg->options,
				  cfg->log_resolution);
	if (!status)
		hw->fwlog_cfg = *cfg;

	return status;
}

/**
 * ice_aq_fwlog_register - Register PF for firmware logging events (0xFF31)
 * @hw: pointer to the HW structure
 * @reg: true to register and false to unregister
 */
enum ice_status
ice_aq_fwlog_register(struct ice_hw *hw, bool reg)
{
	struct ice_aq_desc desc;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_fw_logs_register);

	if (reg)
		desc.params.fw_log.cmd_flags = ICE_AQC_FW_LOG_AQ_REGISTER;

	return ice_aq_send_cmd(hw, &desc, NULL, 0, NULL);
}

/**
 * ice_fwlog_register - Register the PF for firmware logging
 * @hw: pointer to the HW structure
 *
 * After this call the PF will start to receive firmware logging based on the
 * configuration set in ice_fwlog_set.
 */
enum ice_status
ice_fwlog_register(struct ice_hw *hw)
{
	enum ice_status status;

	if (!ice_fwlog_supported(hw))
		return ICE_ERR_NOT_SUPPORTED;

	status = ice_aq_fwlog_register(hw, true);
	if (status)
		DNPRINTF(ICE_DBG_FW_LOG, "Failed to register for firmware "
		    "logging events over ARQ\n");
	else
		hw->fwlog_cfg.options |= ICE_FWLOG_OPTION_IS_REGISTERED;

	return status;
}

/**
 * ice_fwlog_unregister - Unregister the PF from firmware logging
 * @hw: pointer to the HW structure
 */
enum ice_status
ice_fwlog_unregister(struct ice_hw *hw)
{
	enum ice_status status;

	if (!ice_fwlog_supported(hw))
		return ICE_ERR_NOT_SUPPORTED;

	status = ice_aq_fwlog_register(hw, false);
	if (status)
		DNPRINTF(ICE_DBG_FW_LOG, "Failed to unregister from "
		    "firmware logging events over ARQ\n");
	else
		hw->fwlog_cfg.options &= ~ICE_FWLOG_OPTION_IS_REGISTERED;

	return status;
}

/**
 * ice_acquire_nvm - Generic request for acquiring the NVM ownership
 * @hw: pointer to the HW structure
 * @access: NVM access type (read or write)
 *
 * This function will request NVM ownership.
 */
enum ice_status
ice_acquire_nvm(struct ice_hw *hw, enum ice_aq_res_access_type access)
{
	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);
	if (hw->flash.blank_nvm_mode)
		return ICE_SUCCESS;

	return ice_acquire_res(hw, ICE_NVM_RES_ID, access, ICE_NVM_TIMEOUT);
}

/**
 * ice_release_nvm - Generic request for releasing the NVM ownership
 * @hw: pointer to the HW structure
 *
 * This function will release NVM ownership.
 */
void
ice_release_nvm(struct ice_hw *hw)
{
	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);

	if (hw->flash.blank_nvm_mode)
		return;

	ice_release_res(hw, ICE_NVM_RES_ID);
}

/**
 * ice_get_flash_bank_offset - Get offset into requested flash bank
 * @hw: pointer to the HW structure
 * @bank: whether to read from the active or inactive flash bank
 * @module: the module to read from
 *
 * Based on the module, lookup the module offset from the beginning of the
 * flash.
 *
 * Returns the flash offset. Note that a value of zero is invalid and must be
 * treated as an error.
 */
uint32_t ice_get_flash_bank_offset(struct ice_hw *hw,
    enum ice_bank_select bank, uint16_t module)
{
	struct ice_bank_info *banks = &hw->flash.banks;
	enum ice_flash_bank active_bank;
	bool second_bank_active;
	uint32_t offset, size;

	switch (module) {
	case ICE_SR_1ST_NVM_BANK_PTR:
		offset = banks->nvm_ptr;
		size = banks->nvm_size;
		active_bank = banks->nvm_bank;
		break;
	case ICE_SR_1ST_OROM_BANK_PTR:
		offset = banks->orom_ptr;
		size = banks->orom_size;
		active_bank = banks->orom_bank;
		break;
	case ICE_SR_NETLIST_BANK_PTR:
		offset = banks->netlist_ptr;
		size = banks->netlist_size;
		active_bank = banks->netlist_bank;
		break;
	default:
		DNPRINTF(ICE_DBG_NVM,
		    "Unexpected value for flash module: 0x%04x\n", module);
		return 0;
	}

	switch (active_bank) {
	case ICE_1ST_FLASH_BANK:
		second_bank_active = false;
		break;
	case ICE_2ND_FLASH_BANK:
		second_bank_active = true;
		break;
	default:
		DNPRINTF(ICE_DBG_NVM,
		    "Unexpected value for active flash bank: %u\n",
		    active_bank);
		return 0;
	}

	/* The second flash bank is stored immediately following the first
	 * bank. Based on whether the 1st or 2nd bank is active, and whether
	 * we want the active or inactive bank, calculate the desired offset.
	 */
	switch (bank) {
	case ICE_ACTIVE_FLASH_BANK:
		return offset + (second_bank_active ? size : 0);
	case ICE_INACTIVE_FLASH_BANK:
		return offset + (second_bank_active ? 0 : size);
	}

	DNPRINTF(ICE_DBG_NVM,
	    "Unexpected value for flash bank selection: %u\n", bank);

	return 0;
}

/**
 * ice_aq_read_nvm
 * @hw: pointer to the HW struct
 * @module_typeid: module pointer location in words from the NVM beginning
 * @offset: byte offset from the module beginning
 * @length: length of the section to be read (in bytes from the offset)
 * @data: command buffer (size [bytes] = length)
 * @last_command: tells if this is the last command in a series
 * @read_shadow_ram: tell if this is a shadow RAM read
 * @cd: pointer to command details structure or NULL
 *
 * Read the NVM using the admin queue commands (0x0701)
 */
enum ice_status
ice_aq_read_nvm(struct ice_hw *hw, uint16_t module_typeid, uint32_t offset,
    uint16_t length, void *data, bool last_command, bool read_shadow_ram,
    struct ice_sq_cd *cd)
{
	struct ice_aq_desc desc;
	struct ice_aqc_nvm *cmd;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);

	cmd = &desc.params.nvm;

	if (offset > ICE_AQC_NVM_MAX_OFFSET)
		return ICE_ERR_PARAM;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_nvm_read);

	if (!read_shadow_ram && module_typeid == ICE_AQC_NVM_START_POINT)
		cmd->cmd_flags |= ICE_AQC_NVM_FLASH_ONLY;

	/* If this is the last command in a series, set the proper flag. */
	if (last_command)
		cmd->cmd_flags |= ICE_AQC_NVM_LAST_CMD;
	cmd->module_typeid = htole16(module_typeid);
	cmd->offset_low = htole16(offset & 0xFFFF);
	cmd->offset_high = (offset >> 16) & 0xFF;
	cmd->length = htole16(length);

	return ice_aq_send_cmd(hw, &desc, data, length, cd);
}

/**
 * ice_read_flat_nvm - Read portion of NVM by flat offset
 * @hw: pointer to the HW struct
 * @offset: offset from beginning of NVM
 * @length: (in) number of bytes to read; (out) number of bytes actually read
 * @data: buffer to return data in (sized to fit the specified length)
 * @read_shadow_ram: if true, read from shadow RAM instead of NVM
 *
 * Reads a portion of the NVM, as a flat memory space. This function correctly
 * breaks read requests across Shadow RAM sectors and ensures that no single
 * read request exceeds the maximum 4KB read for a single AdminQ command.
 *
 * Returns a status code on failure. Note that the data pointer may be
 * partially updated if some reads succeed before a failure.
 */
enum ice_status
ice_read_flat_nvm(struct ice_hw *hw, uint32_t offset, uint32_t *length,
    uint8_t *data, bool read_shadow_ram)
{
	enum ice_status status;
	uint32_t inlen = *length;
	uint32_t bytes_read = 0;
	bool last_cmd;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);

	*length = 0;

	/* Verify the length of the read if this is for the Shadow RAM */
	if (read_shadow_ram && ((offset + inlen) > (hw->flash.sr_words * 2u))) {
		DNPRINTF(ICE_DBG_NVM,
		    "NVM error: requested data is beyond Shadow RAM limit\n");
		return ICE_ERR_PARAM;
	}

	do {
		uint32_t read_size, sector_offset;

		/* ice_aq_read_nvm cannot read more than 4KB at a time.
		 * Additionally, a read from the Shadow RAM may not cross over
		 * a sector boundary. Conveniently, the sector size is also
		 * 4KB.
		 */
		sector_offset = offset % ICE_AQ_MAX_BUF_LEN;
		read_size = MIN(ICE_AQ_MAX_BUF_LEN - sector_offset,
		    inlen - bytes_read);

		last_cmd = !(bytes_read + read_size < inlen);

		/* ice_aq_read_nvm takes the length as a u16. Our read_size is
		 * calculated using a u32, but the ICE_AQ_MAX_BUF_LEN maximum
		 * size guarantees that it will fit within the 2 bytes.
		 */
		status = ice_aq_read_nvm(hw, ICE_AQC_NVM_START_POINT,
		    offset, (uint16_t)read_size, data + bytes_read, last_cmd,
		    read_shadow_ram, NULL);
		if (status)
			break;

		bytes_read += read_size;
		offset += read_size;
	} while (!last_cmd);

	*length = bytes_read;
	return status;
}

/**
 * ice_discover_flash_size - Discover the available flash size
 * @hw: pointer to the HW struct
 *
 * The device flash could be up to 16MB in size. However, it is possible that
 * the actual size is smaller. Use bisection to determine the accessible size
 * of flash memory.
 */
enum ice_status
ice_discover_flash_size(struct ice_hw *hw)
{
	uint32_t min_size = 0, max_size = ICE_AQC_NVM_MAX_OFFSET + 1;
	enum ice_status status;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);

	status = ice_acquire_nvm(hw, ICE_RES_READ);
	if (status)
		return status;

	while ((max_size - min_size) > 1) {
		uint32_t offset = (max_size + min_size) / 2;
		uint32_t len = 1;
		uint8_t data;

		status = ice_read_flat_nvm(hw, offset, &len, &data, false);
		if (status == ICE_ERR_AQ_ERROR &&
		    hw->adminq.sq_last_status == ICE_AQ_RC_EINVAL) {
			DNPRINTF(ICE_DBG_NVM,
			    "%s: New upper bound of %u bytes\n", __func__,
			    offset);
			status = ICE_SUCCESS;
			max_size = offset;
		} else if (!status) {
			DNPRINTF(ICE_DBG_NVM,
			    "%s: New lower bound of %u bytes\n", __func__,
			    offset);
			min_size = offset;
		} else {
			/* an unexpected error occurred */
			goto err_read_flat_nvm;
		}
	}

	DNPRINTF(ICE_DBG_NVM, "Predicted flash size is %u bytes\n", max_size);

	hw->flash.flash_size = max_size;

err_read_flat_nvm:
	ice_release_nvm(hw);

	return status;
}

/**
 * ice_read_sr_word_aq - Reads Shadow RAM via AQ
 * @hw: pointer to the HW structure
 * @offset: offset of the Shadow RAM word to read (0x000000 - 0x001FFF)
 * @data: word read from the Shadow RAM
 *
 * Reads one 16 bit word from the Shadow RAM using ice_read_flat_nvm.
 */
enum ice_status
ice_read_sr_word_aq(struct ice_hw *hw, uint16_t offset, uint16_t *data)
{
	uint32_t bytes = sizeof(uint16_t);
	enum ice_status status;
	uint16_t data_local;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);

	/* Note that ice_read_flat_nvm checks if the read is past the Shadow
	 * RAM size, and ensures we don't read across a Shadow RAM sector
	 * boundary
	 */
	status = ice_read_flat_nvm(hw, offset * sizeof(uint16_t), &bytes,
				   (uint8_t *)&data_local, true);
	if (status)
		return status;

	*data = le16toh(data_local);
	return ICE_SUCCESS;
}

/**
 * ice_read_sr_word - Reads Shadow RAM word and acquire NVM if necessary
 * @hw: pointer to the HW structure
 * @offset: offset of the Shadow RAM word to read (0x000000 - 0x001FFF)
 * @data: word read from the Shadow RAM
 *
 * Reads one 16 bit word from the Shadow RAM using the ice_read_sr_word_aq.
 */
enum ice_status
ice_read_sr_word(struct ice_hw *hw, uint16_t offset, uint16_t *data)
{
	enum ice_status status;

	status = ice_acquire_nvm(hw, ICE_RES_READ);
	if (!status) {
		status = ice_read_sr_word_aq(hw, offset, data);
		ice_release_nvm(hw);
	}

	return status;
}

/**
 * ice_read_sr_pointer - Read the value of a Shadow RAM pointer word
 * @hw: pointer to the HW structure
 * @offset: the word offset of the Shadow RAM word to read
 * @pointer: pointer value read from Shadow RAM
 *
 * Read the given Shadow RAM word, and convert it to a pointer value specified
 * in bytes. This function assumes the specified offset is a valid pointer
 * word.
 *
 * Each pointer word specifies whether it is stored in word size or 4KB
 * sector size by using the highest bit. The reported pointer value will be in
 * bytes, intended for flat NVM reads.
 */
enum ice_status
ice_read_sr_pointer(struct ice_hw *hw, uint16_t offset, uint32_t *pointer)
{
	enum ice_status status;
	uint16_t value;

	status = ice_read_sr_word(hw, offset, &value);
	if (status)
		return status;

	/* Determine if the pointer is in 4KB or word units */
	if (value & ICE_SR_NVM_PTR_4KB_UNITS)
		*pointer = (value & ~ICE_SR_NVM_PTR_4KB_UNITS) * 4 * 1024;
	else
		*pointer = value * 2;

	return ICE_SUCCESS;
}

/**
 * ice_read_sr_area_size - Read an area size from a Shadow RAM word
 * @hw: pointer to the HW structure
 * @offset: the word offset of the Shadow RAM to read
 * @size: size value read from the Shadow RAM
 *
 * Read the given Shadow RAM word, and convert it to an area size value
 * specified in bytes. This function assumes the specified offset is a valid
 * area size word.
 *
 * Each area size word is specified in 4KB sector units. This function reports
 * the size in bytes, intended for flat NVM reads.
 */
enum ice_status
ice_read_sr_area_size(struct ice_hw *hw, uint16_t offset, uint32_t *size)
{
	enum ice_status status;
	uint16_t value;

	status = ice_read_sr_word(hw, offset, &value);
	if (status)
		return status;

	/* Area sizes are always specified in 4KB units */
	*size = value * 4 * 1024;

	return ICE_SUCCESS;
}

/**
 * ice_determine_active_flash_banks - Discover active bank for each module
 * @hw: pointer to the HW struct
 *
 * Read the Shadow RAM control word and determine which banks are active for
 * the NVM, OROM, and Netlist modules. Also read and calculate the associated
 * pointer and size. These values are then cached into the ice_flash_info
 * structure for later use in order to calculate the correct offset to read
 * from the active module.
 */
enum ice_status
ice_determine_active_flash_banks(struct ice_hw *hw)
{
	struct ice_bank_info *banks = &hw->flash.banks;
	enum ice_status status;
	uint16_t ctrl_word;

	status = ice_read_sr_word(hw, ICE_SR_NVM_CTRL_WORD, &ctrl_word);
	if (status) {
		DNPRINTF(ICE_DBG_NVM,
		    "Failed to read the Shadow RAM control word\n");
		return status;
	}

	/* Check that the control word indicates validity */
	if ((ctrl_word & ICE_SR_CTRL_WORD_1_M) >> ICE_SR_CTRL_WORD_1_S !=
	    ICE_SR_CTRL_WORD_VALID) {
		DNPRINTF(ICE_DBG_NVM, "Shadow RAM control word is invalid\n");
		return ICE_ERR_CFG;
	}

	if (!(ctrl_word & ICE_SR_CTRL_WORD_NVM_BANK))
		banks->nvm_bank = ICE_1ST_FLASH_BANK;
	else
		banks->nvm_bank = ICE_2ND_FLASH_BANK;

	if (!(ctrl_word & ICE_SR_CTRL_WORD_OROM_BANK))
		banks->orom_bank = ICE_1ST_FLASH_BANK;
	else
		banks->orom_bank = ICE_2ND_FLASH_BANK;

	if (!(ctrl_word & ICE_SR_CTRL_WORD_NETLIST_BANK))
		banks->netlist_bank = ICE_1ST_FLASH_BANK;
	else
		banks->netlist_bank = ICE_2ND_FLASH_BANK;

	status = ice_read_sr_pointer(hw, ICE_SR_1ST_NVM_BANK_PTR, &banks->nvm_ptr);
	if (status) {
		DNPRINTF(ICE_DBG_NVM, "Failed to read NVM bank pointer\n");
		return status;
	}

	status = ice_read_sr_area_size(hw, ICE_SR_NVM_BANK_SIZE, &banks->nvm_size);
	if (status) {
		DNPRINTF(ICE_DBG_NVM, "Failed to read NVM bank area size\n");
		return status;
	}

	status = ice_read_sr_pointer(hw, ICE_SR_1ST_OROM_BANK_PTR, &banks->orom_ptr);
	if (status) {
		DNPRINTF(ICE_DBG_NVM, "Failed to read OROM bank pointer\n");
		return status;
	}

	status = ice_read_sr_area_size(hw, ICE_SR_OROM_BANK_SIZE, &banks->orom_size);
	if (status) {
		DNPRINTF(ICE_DBG_NVM, "Failed to read OROM bank area size\n");
		return status;
	}

	status = ice_read_sr_pointer(hw, ICE_SR_NETLIST_BANK_PTR, &banks->netlist_ptr);
	if (status) {
		DNPRINTF(ICE_DBG_NVM, "Failed to read Netlist bank pointer\n");
		return status;
	}

	status = ice_read_sr_area_size(hw, ICE_SR_NETLIST_BANK_SIZE,
	    &banks->netlist_size);
	if (status) {
		DNPRINTF(ICE_DBG_NVM,
		    "Failed to read Netlist bank area size\n");
		return status;
	}

	return ICE_SUCCESS;
}

/**
 * ice_read_flash_module - Read a word from one of the main NVM modules
 * @hw: pointer to the HW structure
 * @bank: which bank of the module to read
 * @module: the module to read
 * @offset: the offset into the module in bytes
 * @data: storage for the word read from the flash
 * @length: bytes of data to read
 *
 * Read data from the specified flash module. The bank parameter indicates
 * whether or not to read from the active bank or the inactive bank of that
 * module.
 *
 * The word will be read using flat NVM access, and relies on the
 * hw->flash.banks data being setup by ice_determine_active_flash_banks()
 * during initialization.
 */
enum ice_status
ice_read_flash_module(struct ice_hw *hw, enum ice_bank_select bank,
    uint16_t module, uint32_t offset, uint8_t *data, uint32_t length)
{
	enum ice_status status;
	uint32_t start;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);

	start = ice_get_flash_bank_offset(hw, bank, module);
	if (!start) {
		DNPRINTF(ICE_DBG_NVM,
		    "Unable to calculate flash bank offset for module 0x%04x\n",
		    module);
		return ICE_ERR_PARAM;
	}

	status = ice_acquire_nvm(hw, ICE_RES_READ);
	if (status)
		return status;

	status = ice_read_flat_nvm(hw, start + offset, &length, data, false);

	ice_release_nvm(hw);

	return status;
}

/**
 * ice_read_nvm_module - Read from the active main NVM module
 * @hw: pointer to the HW structure
 * @bank: whether to read from active or inactive NVM module
 * @offset: offset into the NVM module to read, in words
 * @data: storage for returned word value
 *
 * Read the specified word from the active NVM module. This includes the CSS
 * header at the start of the NVM module.
 */
enum ice_status
ice_read_nvm_module(struct ice_hw *hw, enum ice_bank_select bank,
    uint32_t offset, uint16_t *data)
{
	enum ice_status status;
	uint16_t data_local;

	status = ice_read_flash_module(hw, bank, ICE_SR_1ST_NVM_BANK_PTR,
	    offset * sizeof(uint16_t), (uint8_t *)&data_local,
	    sizeof(uint16_t));
	if (!status)
		*data = le16toh(data_local);

	return status;
}

/**
 * ice_get_nvm_css_hdr_len - Read the CSS header length from the NVM CSS header
 * @hw: pointer to the HW struct
 * @bank: whether to read from the active or inactive flash bank
 * @hdr_len: storage for header length in words
 *
 * Read the CSS header length from the NVM CSS header and add the Authentication
 * header size, and then convert to words.
 */
enum ice_status
ice_get_nvm_css_hdr_len(struct ice_hw *hw, enum ice_bank_select bank,
    uint32_t *hdr_len)
{
	uint16_t hdr_len_l, hdr_len_h;
	enum ice_status status;
	uint32_t hdr_len_dword;

	status = ice_read_nvm_module(hw, bank, ICE_NVM_CSS_HDR_LEN_L,
	    &hdr_len_l);
	if (status)
		return status;

	status = ice_read_nvm_module(hw, bank, ICE_NVM_CSS_HDR_LEN_H,
	    &hdr_len_h);
	if (status)
		return status;

	/* CSS header length is in DWORD, so convert to words and add
	 * authentication header size
	 */
	hdr_len_dword = hdr_len_h << 16 | hdr_len_l;
	*hdr_len = (hdr_len_dword * 2) + ICE_NVM_AUTH_HEADER_LEN;

	return ICE_SUCCESS;
}

/**
 * ice_get_nvm_srev - Read the security revision from the NVM CSS header
 * @hw: pointer to the HW struct
 * @bank: whether to read from the active or inactive flash bank
 * @srev: storage for security revision
 *
 * Read the security revision out of the CSS header of the active NVM module
 * bank.
 */
enum ice_status ice_get_nvm_srev(struct ice_hw *hw, enum ice_bank_select bank,
    uint32_t *srev)
{
	enum ice_status status;
	uint16_t srev_l, srev_h;

	status = ice_read_nvm_module(hw, bank, ICE_NVM_CSS_SREV_L, &srev_l);
	if (status)
		return status;

	status = ice_read_nvm_module(hw, bank, ICE_NVM_CSS_SREV_H, &srev_h);
	if (status)
		return status;

	*srev = srev_h << 16 | srev_l;

	return ICE_SUCCESS;
}

/**
 * ice_read_nvm_sr_copy - Read a word from the Shadow RAM copy in the NVM bank
 * @hw: pointer to the HW structure
 * @bank: whether to read from the active or inactive NVM module
 * @offset: offset into the Shadow RAM copy to read, in words
 * @data: storage for returned word value
 *
 * Read the specified word from the copy of the Shadow RAM found in the
 * specified NVM module.
 */
enum ice_status
ice_read_nvm_sr_copy(struct ice_hw *hw, enum ice_bank_select bank,
    uint32_t offset, uint16_t *data)
{
	enum ice_status status;
	uint32_t hdr_len;

	status = ice_get_nvm_css_hdr_len(hw, bank, &hdr_len);
	if (status)
		return status;

	hdr_len = roundup(hdr_len, 32);

	return ice_read_nvm_module(hw, bank, hdr_len + offset, data);
}

/**
 * ice_get_nvm_ver_info - Read NVM version information
 * @hw: pointer to the HW struct
 * @bank: whether to read from the active or inactive flash bank
 * @nvm: pointer to NVM info structure
 *
 * Read the NVM EETRACK ID and map version of the main NVM image bank, filling
 * in the NVM info structure.
 */
enum ice_status
ice_get_nvm_ver_info(struct ice_hw *hw, enum ice_bank_select bank,
    struct ice_nvm_info *nvm)
{
	uint16_t eetrack_lo, eetrack_hi, ver;
	enum ice_status status;

	status = ice_read_nvm_sr_copy(hw, bank, ICE_SR_NVM_DEV_STARTER_VER,
	    &ver);
	if (status) {
		DNPRINTF(ICE_DBG_NVM, "Failed to read DEV starter version.\n");
		return status;
	}

	nvm->major = (ver & ICE_NVM_VER_HI_MASK) >> ICE_NVM_VER_HI_SHIFT;
	nvm->minor = (ver & ICE_NVM_VER_LO_MASK) >> ICE_NVM_VER_LO_SHIFT;

	status = ice_read_nvm_sr_copy(hw, bank, ICE_SR_NVM_EETRACK_LO,
	    &eetrack_lo);
	if (status) {
		DNPRINTF(ICE_DBG_NVM, "Failed to read EETRACK lo.\n");
		return status;
	}
	status = ice_read_nvm_sr_copy(hw, bank, ICE_SR_NVM_EETRACK_HI,
	     &eetrack_hi);
	if (status) {
		DNPRINTF(ICE_DBG_NVM, "Failed to read EETRACK hi.\n");
		return status;
	}

	nvm->eetrack = (eetrack_hi << 16) | eetrack_lo;

	status = ice_get_nvm_srev(hw, bank, &nvm->srev);
	if (status)
		DNPRINTF(ICE_DBG_NVM,
		    "Failed to read NVM security revision.\n");

	return ICE_SUCCESS;
}

/**
 * ice_read_orom_module - Read from the active Option ROM module
 * @hw: pointer to the HW structure
 * @bank: whether to read from active or inactive OROM module
 * @offset: offset into the OROM module to read, in words
 * @data: storage for returned word value
 *
 * Read the specified word from the active Option ROM module of the flash.
 * Note that unlike the NVM module, the CSS data is stored at the end of the
 * module instead of at the beginning.
 */
enum ice_status
ice_read_orom_module(struct ice_hw *hw, enum ice_bank_select bank,
    uint32_t offset, uint16_t *data)
{
	enum ice_status status;
	uint16_t data_local;

	status = ice_read_flash_module(hw, bank, ICE_SR_1ST_OROM_BANK_PTR,
	    offset * sizeof(uint16_t), (uint8_t *)&data_local,
	    sizeof(uint16_t));
	if (!status)
		*data = le16toh(data_local);

	return status;
}

/**
 * ice_get_orom_srev - Read the security revision from the OROM CSS header
 * @hw: pointer to the HW struct
 * @bank: whether to read from active or inactive flash module
 * @srev: storage for security revision
 *
 * Read the security revision out of the CSS header of the active OROM module
 * bank.
 */
enum ice_status
ice_get_orom_srev(struct ice_hw *hw, enum ice_bank_select bank, uint32_t *srev)
{
	uint32_t orom_size_word = hw->flash.banks.orom_size / 2;
	enum ice_status status;
	uint16_t srev_l, srev_h;
	uint32_t css_start;
	uint32_t hdr_len;

	status = ice_get_nvm_css_hdr_len(hw, bank, &hdr_len);
	if (status)
		return status;

	if (orom_size_word < hdr_len) {
		DNPRINTF(ICE_DBG_NVM, "Unexpected Option ROM Size of %u\n",
		    hw->flash.banks.orom_size);
		return ICE_ERR_CFG;
	}

	/* calculate how far into the Option ROM the CSS header starts. Note
	 * that ice_read_orom_module takes a word offset
	 */
	css_start = orom_size_word - hdr_len;
	status = ice_read_orom_module(hw, bank, css_start + ICE_NVM_CSS_SREV_L,
	    &srev_l);
	if (status)
		return status;

	status = ice_read_orom_module(hw, bank, css_start + ICE_NVM_CSS_SREV_H,
	    &srev_h);
	if (status)
		return status;

	*srev = srev_h << 16 | srev_l;

	return ICE_SUCCESS;
}

/**
 * ice_get_orom_civd_data - Get the combo version information from Option ROM
 * @hw: pointer to the HW struct
 * @bank: whether to read from the active or inactive flash module
 * @civd: storage for the Option ROM CIVD data.
 *
 * Searches through the Option ROM flash contents to locate the CIVD data for
 * the image.
 */
enum ice_status
ice_get_orom_civd_data(struct ice_hw *hw, enum ice_bank_select bank,
		       struct ice_orom_civd_info *civd)
{
	uint8_t *orom_data;
	enum ice_status status;
	uint32_t offset;

	/* The CIVD section is located in the Option ROM aligned to 512 bytes.
	 * The first 4 bytes must contain the ASCII characters "$CIV".
	 * A simple modulo 256 sum of all of the bytes of the structure must
	 * equal 0.
	 *
	 * The exact location is unknown and varies between images but is
	 * usually somewhere in the middle of the bank. We need to scan the
	 * Option ROM bank to locate it.
	 *
	 * It's significantly faster to read the entire Option ROM up front
	 * using the maximum page size, than to read each possible location
	 * with a separate firmware command.
	 */
	orom_data = (uint8_t *)ice_calloc(hw, hw->flash.banks.orom_size,
	    sizeof(uint8_t));
	if (!orom_data)
		return ICE_ERR_NO_MEMORY;

	status = ice_read_flash_module(hw, bank, ICE_SR_1ST_OROM_BANK_PTR, 0,
	    orom_data, hw->flash.banks.orom_size);
	if (status) {
		DNPRINTF(ICE_DBG_NVM, "Unable to read Option ROM data\n");
		goto exit_error;
	}

	/* Scan the memory buffer to locate the CIVD data section */
	for (offset = 0; (offset + 512) <= hw->flash.banks.orom_size;
	    offset += 512) {
		struct ice_orom_civd_info *tmp;
		uint8_t sum = 0, i;

		tmp = (struct ice_orom_civd_info *)&orom_data[offset];

		/* Skip forward until we find a matching signature */
		if (memcmp("$CIV", tmp->signature, sizeof(tmp->signature)) != 0)
			continue;

		DNPRINTF(ICE_DBG_NVM, "Found CIVD section at offset %u\n",
			  offset);

		/* Verify that the simple checksum is zero */
		for (i = 0; i < sizeof(*tmp); i++)
			sum += ((uint8_t *)tmp)[i];

		if (sum) {
			DNPRINTF(ICE_DBG_NVM,
			    "Found CIVD data with invalid checksum of %u\n",
			    sum);
			status = ICE_ERR_NVM;
			goto exit_error;
		}

		*civd = *tmp;
		ice_free(hw, orom_data);
		return ICE_SUCCESS;
	}

	status = ICE_ERR_NVM;
	DNPRINTF(ICE_DBG_NVM,
	    "Unable to locate CIVD data within the Option ROM\n");

exit_error:
	ice_free(hw, orom_data);
	return status;
}

/**
 * ice_get_orom_ver_info - Read Option ROM version information
 * @hw: pointer to the HW struct
 * @bank: whether to read from the active or inactive flash module
 * @orom: pointer to Option ROM info structure
 *
 * Read Option ROM version and security revision from the Option ROM flash
 * section.
 */
enum ice_status
ice_get_orom_ver_info(struct ice_hw *hw, enum ice_bank_select bank,
    struct ice_orom_info *orom)
{
	struct ice_orom_civd_info civd;
	enum ice_status status;
	uint32_t combo_ver;

	status = ice_get_orom_civd_data(hw, bank, &civd);
	if (status) {
		DNPRINTF(ICE_DBG_NVM,
		    "Failed to locate valid Option ROM CIVD data\n");
		return status;
	}

	combo_ver = le32toh(civd.combo_ver);

	orom->major = (uint8_t)((combo_ver & ICE_OROM_VER_MASK) >>
	    ICE_OROM_VER_SHIFT);
	orom->patch = (uint8_t)(combo_ver & ICE_OROM_VER_PATCH_MASK);
	orom->build = (uint16_t)((combo_ver & ICE_OROM_VER_BUILD_MASK) >>
	    ICE_OROM_VER_BUILD_SHIFT);

	status = ice_get_orom_srev(hw, bank, &orom->srev);
	if (status) {
		DNPRINTF(ICE_DBG_NVM,
		    "Failed to read Option ROM security revision.\n");
		return status;
	}

	return ICE_SUCCESS;
}

/**
 * ice_read_netlist_module - Read data from the netlist module area
 * @hw: pointer to the HW structure
 * @bank: whether to read from the active or inactive module
 * @offset: offset into the netlist to read from
 * @data: storage for returned word value
 *
 * Read a word from the specified netlist bank.
 */
enum ice_status
ice_read_netlist_module(struct ice_hw *hw, enum ice_bank_select bank,
    uint32_t offset, uint16_t *data)
{
	enum ice_status status;
	uint16_t data_local;

	status = ice_read_flash_module(hw, bank, ICE_SR_NETLIST_BANK_PTR,
	    offset * sizeof(uint16_t), (uint8_t *)&data_local,
	    sizeof(uint16_t));
	if (!status)
		*data = le16toh(data_local);

	return status;
}

/**
 * ice_get_netlist_info
 * @hw: pointer to the HW struct
 * @bank: whether to read from the active or inactive flash bank
 * @netlist: pointer to netlist version info structure
 *
 * Get the netlist version information from the requested bank. Reads the Link
 * Topology section to find the Netlist ID block and extract the relevant
 * information into the netlist version structure.
 */
enum ice_status
ice_get_netlist_info(struct ice_hw *hw, enum ice_bank_select bank,
    struct ice_netlist_info *netlist)
{
	uint16_t module_id, length, node_count, i;
	enum ice_status status;
	uint16_t *id_blk;

	status = ice_read_netlist_module(hw, bank, ICE_NETLIST_TYPE_OFFSET,
	    &module_id);
	if (status)
		return status;

	if (module_id != ICE_NETLIST_LINK_TOPO_MOD_ID) {
		DNPRINTF(ICE_DBG_NVM,
		    "Expected netlist module_id ID of 0x%04x, but got 0x%04x\n",
		    ICE_NETLIST_LINK_TOPO_MOD_ID, module_id);
		return ICE_ERR_NVM;
	}

	status = ice_read_netlist_module(hw, bank, ICE_LINK_TOPO_MODULE_LEN,
	    &length);
	if (status)
		return status;

	if (length < ICE_NETLIST_ID_BLK_SIZE) {
		DNPRINTF(ICE_DBG_NVM, "Netlist Link Topology module too small. "
		    "Expected at least %u words, but got %u words.\n",
		    ICE_NETLIST_ID_BLK_SIZE, length);
		return ICE_ERR_NVM;
	}

	status = ice_read_netlist_module(hw, bank, ICE_LINK_TOPO_NODE_COUNT,
	    &node_count);
	if (status)
		return status;
	node_count &= ICE_LINK_TOPO_NODE_COUNT_M;

	id_blk = (uint16_t *)ice_calloc(hw, ICE_NETLIST_ID_BLK_SIZE,
	    sizeof(*id_blk));
	if (!id_blk)
		return ICE_ERR_NO_MEMORY;

	/* Read out the entire Netlist ID Block at once. */
	status = ice_read_flash_module(hw, bank, ICE_SR_NETLIST_BANK_PTR,
	    ICE_NETLIST_ID_BLK_OFFSET(node_count) * sizeof(uint16_t),
	    (uint8_t *)id_blk, ICE_NETLIST_ID_BLK_SIZE * sizeof(uint16_t));
	if (status)
		goto exit_error;

	for (i = 0; i < ICE_NETLIST_ID_BLK_SIZE; i++)
		id_blk[i] = le16toh(((uint16_t *)id_blk)[i]);

	netlist->major = id_blk[ICE_NETLIST_ID_BLK_MAJOR_VER_HIGH] << 16 |
			 id_blk[ICE_NETLIST_ID_BLK_MAJOR_VER_LOW];
	netlist->minor = id_blk[ICE_NETLIST_ID_BLK_MINOR_VER_HIGH] << 16 |
			 id_blk[ICE_NETLIST_ID_BLK_MINOR_VER_LOW];
	netlist->type = id_blk[ICE_NETLIST_ID_BLK_TYPE_HIGH] << 16 |
			id_blk[ICE_NETLIST_ID_BLK_TYPE_LOW];
	netlist->rev = id_blk[ICE_NETLIST_ID_BLK_REV_HIGH] << 16 |
		       id_blk[ICE_NETLIST_ID_BLK_REV_LOW];
	netlist->cust_ver = id_blk[ICE_NETLIST_ID_BLK_CUST_VER];
	/* Read the left most 4 bytes of SHA */
	netlist->hash = id_blk[ICE_NETLIST_ID_BLK_SHA_HASH_WORD(15)] << 16 |
			id_blk[ICE_NETLIST_ID_BLK_SHA_HASH_WORD(14)];

exit_error:
	ice_free(hw, id_blk);

	return status;
}

/**
 * ice_init_nvm - initializes NVM setting
 * @hw: pointer to the HW struct
 *
 * This function reads and populates NVM settings such as Shadow RAM size,
 * max_timeout, and blank_nvm_mode
 */
enum ice_status
ice_init_nvm(struct ice_hw *hw)
{
	struct ice_flash_info *flash = &hw->flash;
	enum ice_status status;
	uint32_t fla, gens_stat;
	uint8_t sr_size;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);

	/* The SR size is stored regardless of the NVM programming mode
	 * as the blank mode may be used in the factory line.
	 */
	gens_stat = ICE_READ(hw, GLNVM_GENS);
	sr_size = (gens_stat & GLNVM_GENS_SR_SIZE_M) >> GLNVM_GENS_SR_SIZE_S;

	/* Switching to words (sr_size contains power of 2) */
	flash->sr_words = BIT(sr_size) * ICE_SR_WORDS_IN_1KB;

	/* Check if we are in the normal or blank NVM programming mode */
	fla = ICE_READ(hw, GLNVM_FLA);
	if (fla & GLNVM_FLA_LOCKED_M) { /* Normal programming mode */
		flash->blank_nvm_mode = false;
	} else {
		/* Blank programming mode */
		flash->blank_nvm_mode = true;
		DNPRINTF(ICE_DBG_NVM,
		    "NVM init error: unsupported blank mode.\n");
		return ICE_ERR_NVM_BLANK_MODE;
	}

	status = ice_discover_flash_size(hw);
	if (status) {
		DNPRINTF(ICE_DBG_NVM, "NVM init error: failed to discover "
		    "flash size.\n");
		return status;
	}

	status = ice_determine_active_flash_banks(hw);
	if (status) {
		DNPRINTF(ICE_DBG_NVM, "Failed to determine active flash "
		    "banks.\n");
		return status;
	}

	status = ice_get_nvm_ver_info(hw, ICE_ACTIVE_FLASH_BANK, &flash->nvm);
	if (status) {
		DNPRINTF(ICE_DBG_INIT, "Failed to read NVM info.\n");
		return status;
	}

	status = ice_get_orom_ver_info(hw, ICE_ACTIVE_FLASH_BANK, &flash->orom);
	if (status)
		DNPRINTF(ICE_DBG_INIT, "Failed to read Option ROM info.\n");

	/* read the netlist version information */
	status = ice_get_netlist_info(hw, ICE_ACTIVE_FLASH_BANK,
	    &flash->netlist);
	if (status)
		DNPRINTF(ICE_DBG_INIT, "Failed to read netlist info.\n");

	return ICE_SUCCESS;
}

void
ice_print_rollback_msg(struct ice_hw *hw)
{
	struct ice_softc *sc = hw->hw_sc;
	char nvm_str[ICE_NVM_VER_LEN] = { 0 };
	struct ice_orom_info *orom;
	struct ice_nvm_info *nvm;

	orom = &hw->flash.orom;
	nvm = &hw->flash.nvm;

	snprintf(nvm_str, sizeof(nvm_str), "%x.%02x 0x%x %d.%d.%d",
		 nvm->major, nvm->minor, nvm->eetrack, orom->major,
		 orom->build, orom->patch);
	printf("%s: Firmware rollback mode detected. "
	    "Current version is NVM: %s, FW: %d.%d. "
	    "Device may exhibit limited functionality.\n",
	    sc->sc_dev.dv_xname, nvm_str, hw->fw_maj_ver, hw->fw_min_ver);
}

/**
 * ice_clear_pf_cfg - Clear PF configuration
 * @hw: pointer to the hardware structure
 *
 * Clears any existing PF configuration (VSIs, VSI lists, switch rules, port
 * configuration, flow director filters, etc.).
 */
enum ice_status
ice_clear_pf_cfg(struct ice_hw *hw)
{
	struct ice_aq_desc desc;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_clear_pf_cfg);

	return ice_aq_send_cmd(hw, &desc, NULL, 0, NULL);
}

/**
 * ice_aq_clear_pxe_mode
 * @hw: pointer to the HW struct
 *
 * Tell the firmware that the driver is taking over from PXE (0x0110).
 */
enum ice_status
ice_aq_clear_pxe_mode(struct ice_hw *hw)
{
	struct ice_aq_desc desc;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_clear_pxe_mode);
	desc.params.clear_pxe.rx_cnt = ICE_AQC_CLEAR_PXE_RX_CNT;

	return ice_aq_send_cmd(hw, &desc, NULL, 0, NULL);
}

/**
 * ice_clear_pxe_mode - clear pxe operations mode
 * @hw: pointer to the HW struct
 *
 * Make sure all PXE mode settings are cleared, including things
 * like descriptor fetch/write-back mode.
 */
void
ice_clear_pxe_mode(struct ice_hw *hw)
{
	if (ice_check_sq_alive(hw, &hw->adminq))
		ice_aq_clear_pxe_mode(hw);
}

/**
 * ice_get_num_per_func - determine number of resources per PF
 * @hw: pointer to the HW structure
 * @max: value to be evenly split between each PF
 *
 * Determine the number of valid functions by going through the bitmap returned
 * from parsing capabilities and use this to calculate the number of resources
 * per PF based on the max value passed in.
 */
uint32_t
ice_get_num_per_func(struct ice_hw *hw, uint32_t max)
{
	uint16_t funcs;

#define ICE_CAPS_VALID_FUNCS_M	0xFF
	funcs = ice_popcount16(hw->dev_caps.common_cap.valid_functions &
	    ICE_CAPS_VALID_FUNCS_M);

	if (!funcs)
		return 0;

	return max / funcs;
}

/**
 * ice_print_led_caps - print LED capabilities
 * @hw: pointer to the ice_hw instance
 * @caps: pointer to common caps instance
 * @prefix: string to prefix when printing
 * @dbg: set to indicate debug print
 */
void
ice_print_led_caps(struct ice_hw *hw, struct ice_hw_common_caps *caps,
		   char const *prefix, bool dbg)
{
	struct ice_softc *sc = hw->hw_sc;
	uint8_t i;

	if (dbg)
		DNPRINTF(ICE_DBG_INIT, "%s: led_pin_num = %d\n", prefix,
		    caps->led_pin_num);
	else
		printf("%s: %s: led_pin_num = %d\n", sc->sc_dev.dv_xname,
		   prefix, caps->led_pin_num);

	for (i = 0; i < ICE_MAX_SUPPORTED_GPIO_LED; i++) {
		if (!caps->led[i])
			continue;

		if (dbg)
			DNPRINTF(ICE_DBG_INIT, "%s: led[%d] = %d\n",
			    prefix, i, caps->led[i]);
		else
			printf("%s: %s: led[%d] = %d\n", sc->sc_dev.dv_xname,
			    prefix, i, caps->led[i]);
	}
}

/**
 * ice_print_sdp_caps - print SDP capabilities
 * @hw: pointer to the ice_hw instance
 * @caps: pointer to common caps instance
 * @prefix: string to prefix when printing
 * @dbg: set to indicate debug print
 */
void
ice_print_sdp_caps(struct ice_hw *hw, struct ice_hw_common_caps *caps,
    char const *prefix, bool dbg)
{
	struct ice_softc *sc = hw->hw_sc;
	uint8_t i;

	if (dbg)
		DNPRINTF(ICE_DBG_INIT, "%s: sdp_pin_num = %d\n", prefix,
		    caps->sdp_pin_num);
	else
		printf("%s: %s: sdp_pin_num = %d\n", sc->sc_dev.dv_xname,
		    prefix, caps->sdp_pin_num);

	for (i = 0; i < ICE_MAX_SUPPORTED_GPIO_SDP; i++) {
		if (!caps->sdp[i])
			continue;

		if (dbg)
			DNPRINTF(ICE_DBG_INIT, "%s: sdp[%d] = %d\n",
			    prefix, i, caps->sdp[i]);
		else
			printf("%s: %s: sdp[%d] = %d\n", sc->sc_dev.dv_xname,
			    prefix, i, caps->sdp[i]);
	}
}

/**
 * ice_parse_common_caps - parse common device/function capabilities
 * @hw: pointer to the HW struct
 * @caps: pointer to common capabilities structure
 * @elem: the capability element to parse
 * @prefix: message prefix for tracing capabilities
 *
 * Given a capability element, extract relevant details into the common
 * capability structure.
 *
 * Returns: true if the capability matches one of the common capability ids,
 * false otherwise.
 */
bool
ice_parse_common_caps(struct ice_hw *hw, struct ice_hw_common_caps *caps,
    struct ice_aqc_list_caps_elem *elem, const char *prefix)
{
	uint32_t logical_id = le32toh(elem->logical_id);
	uint32_t phys_id = le32toh(elem->phys_id);
	uint32_t number = le32toh(elem->number);
	uint16_t cap = le16toh(elem->cap);
	bool found = true;

	switch (cap) {
	case ICE_AQC_CAPS_SWITCHING_MODE:
		caps->switching_mode = number;
		DNPRINTF(ICE_DBG_INIT, "%s: switching_mode = %d\n", prefix,
			  caps->switching_mode);
		break;
	case ICE_AQC_CAPS_MANAGEABILITY_MODE:
		caps->mgmt_mode = number;
		caps->mgmt_protocols_mctp = logical_id;
		DNPRINTF(ICE_DBG_INIT, "%s: mgmt_mode = %d\n", prefix,
			  caps->mgmt_mode);
		DNPRINTF(ICE_DBG_INIT, "%s: mgmt_protocols_mctp = %d\n", prefix,
			  caps->mgmt_protocols_mctp);
		break;
	case ICE_AQC_CAPS_OS2BMC:
		caps->os2bmc = number;
		DNPRINTF(ICE_DBG_INIT, "%s: os2bmc = %d\n", prefix,
		    caps->os2bmc);
		break;
	case ICE_AQC_CAPS_VALID_FUNCTIONS:
		caps->valid_functions = number;
		DNPRINTF(ICE_DBG_INIT, "%s: valid_functions (bitmap) = %d\n",
		    prefix, caps->valid_functions);
		break;
	case ICE_AQC_CAPS_SRIOV:
		caps->sr_iov_1_1 = (number == 1);
		DNPRINTF(ICE_DBG_INIT, "%s: sr_iov_1_1 = %d\n", prefix,
			  caps->sr_iov_1_1);
		break;
	case ICE_AQC_CAPS_VMDQ:
		caps->vmdq = (number == 1);
		DNPRINTF(ICE_DBG_INIT, "%s: vmdq = %d\n", prefix, caps->vmdq);
		break;
	case ICE_AQC_CAPS_802_1QBG:
		caps->evb_802_1_qbg = (number == 1);
		DNPRINTF(ICE_DBG_INIT, "%s: evb_802_1_qbg = %d\n", prefix,
		    number);
		break;
	case ICE_AQC_CAPS_802_1BR:
		caps->evb_802_1_qbh = (number == 1);
		DNPRINTF(ICE_DBG_INIT, "%s: evb_802_1_qbh = %d\n", prefix,
		    number);
		break;
	case ICE_AQC_CAPS_DCB:
		caps->dcb = (number == 1);
		caps->active_tc_bitmap = logical_id;
		caps->maxtc = phys_id;
		DNPRINTF(ICE_DBG_INIT, "%s: dcb = %d\n", prefix, caps->dcb);
		DNPRINTF(ICE_DBG_INIT, "%s: active_tc_bitmap = %d\n", prefix,
			  caps->active_tc_bitmap);
		DNPRINTF(ICE_DBG_INIT, "%s: maxtc = %d\n", prefix, caps->maxtc);
		break;
	case ICE_AQC_CAPS_ISCSI:
		caps->iscsi = (number == 1);
		DNPRINTF(ICE_DBG_INIT, "%s: iscsi = %d\n", prefix, caps->iscsi);
		break;
	case ICE_AQC_CAPS_RSS:
		caps->rss_table_size = number;
		caps->rss_table_entry_width = logical_id;
		DNPRINTF(ICE_DBG_INIT, "%s: rss_table_size = %d\n", prefix,
			  caps->rss_table_size);
		DNPRINTF(ICE_DBG_INIT, "%s: rss_table_entry_width = %d\n",
		    prefix, caps->rss_table_entry_width);
		break;
	case ICE_AQC_CAPS_RXQS:
		caps->num_rxq = number;
		caps->rxq_first_id = phys_id;
		DNPRINTF(ICE_DBG_INIT, "%s: num_rxq = %d\n", prefix,
			  caps->num_rxq);
		DNPRINTF(ICE_DBG_INIT, "%s: rxq_first_id = %d\n", prefix,
			  caps->rxq_first_id);
		break;
	case ICE_AQC_CAPS_TXQS:
		caps->num_txq = number;
		caps->txq_first_id = phys_id;
		DNPRINTF(ICE_DBG_INIT, "%s: num_txq = %d\n", prefix,
			  caps->num_txq);
		DNPRINTF(ICE_DBG_INIT, "%s: txq_first_id = %d\n", prefix,
			  caps->txq_first_id);
		break;
	case ICE_AQC_CAPS_MSIX:
		caps->num_msix_vectors = number;
		caps->msix_vector_first_id = phys_id;
		DNPRINTF(ICE_DBG_INIT, "%s: num_msix_vectors = %d\n", prefix,
			  caps->num_msix_vectors);
		DNPRINTF(ICE_DBG_INIT, "%s: msix_vector_first_id = %d\n",
		    prefix, caps->msix_vector_first_id);
		break;
	case ICE_AQC_CAPS_NVM_MGMT:
		caps->sec_rev_disabled =
			(number & ICE_NVM_MGMT_SEC_REV_DISABLED) ?
			true : false;
		DNPRINTF(ICE_DBG_INIT, "%s: sec_rev_disabled = %d\n", prefix,
			  caps->sec_rev_disabled);
		caps->update_disabled =
			(number & ICE_NVM_MGMT_UPDATE_DISABLED) ?
			true : false;
		DNPRINTF(ICE_DBG_INIT, "%s: update_disabled = %d\n", prefix,
			  caps->update_disabled);
		caps->nvm_unified_update =
			(number & ICE_NVM_MGMT_UNIFIED_UPD_SUPPORT) ?
			true : false;
		DNPRINTF(ICE_DBG_INIT, "%s: nvm_unified_update = %d\n", prefix,
			  caps->nvm_unified_update);
		caps->netlist_auth =
			(number & ICE_NVM_MGMT_NETLIST_AUTH_SUPPORT) ?
			true : false;
		DNPRINTF(ICE_DBG_INIT, "%s: netlist_auth = %d\n", prefix,
			  caps->netlist_auth);
		break;
	case ICE_AQC_CAPS_CEM:
		caps->mgmt_cem = (number == 1);
		DNPRINTF(ICE_DBG_INIT, "%s: mgmt_cem = %d\n", prefix,
			  caps->mgmt_cem);
		break;
	case ICE_AQC_CAPS_IWARP:
		caps->iwarp = (number == 1);
		DNPRINTF(ICE_DBG_INIT, "%s: iwarp = %d\n", prefix, caps->iwarp);
		break;
	case ICE_AQC_CAPS_ROCEV2_LAG:
		caps->roce_lag = !!(number & ICE_AQC_BIT_ROCEV2_LAG);
		DNPRINTF(ICE_DBG_INIT, "%s: roce_lag = %d\n",
			  prefix, caps->roce_lag);
		break;
	case ICE_AQC_CAPS_LED:
		if (phys_id < ICE_MAX_SUPPORTED_GPIO_LED) {
			caps->led[phys_id] = true;
			caps->led_pin_num++;
			DNPRINTF(ICE_DBG_INIT, "%s: led[%d] = 1\n", prefix,
			    phys_id);
		}
		break;
	case ICE_AQC_CAPS_SDP:
		if (phys_id < ICE_MAX_SUPPORTED_GPIO_SDP) {
			caps->sdp[phys_id] = true;
			caps->sdp_pin_num++;
			DNPRINTF(ICE_DBG_INIT, "%s: sdp[%d] = 1\n", prefix,
			    phys_id);
		}
		break;
	case ICE_AQC_CAPS_WR_CSR_PROT:
		caps->wr_csr_prot = number;
		caps->wr_csr_prot |= (uint64_t)logical_id << 32;
		DNPRINTF(ICE_DBG_INIT, "%s: wr_csr_prot = 0x%llX\n", prefix,
			  (unsigned long long)caps->wr_csr_prot);
		break;
	case ICE_AQC_CAPS_WOL_PROXY:
		caps->num_wol_proxy_fltr = number;
		caps->wol_proxy_vsi_seid = logical_id;
		caps->apm_wol_support = !!(phys_id & ICE_WOL_SUPPORT_M);
		caps->acpi_prog_mthd = !!(phys_id &
					  ICE_ACPI_PROG_MTHD_M);
		caps->proxy_support = !!(phys_id & ICE_PROXY_SUPPORT_M);
		DNPRINTF(ICE_DBG_INIT, "%s: num_wol_proxy_fltr = %d\n", prefix,
			  caps->num_wol_proxy_fltr);
		DNPRINTF(ICE_DBG_INIT, "%s: wol_proxy_vsi_seid = %d\n", prefix,
			  caps->wol_proxy_vsi_seid);
		DNPRINTF(ICE_DBG_INIT, "%s: apm_wol_support = %d\n",
			  prefix, caps->apm_wol_support);
		break;
	case ICE_AQC_CAPS_MAX_MTU:
		caps->max_mtu = number;
		DNPRINTF(ICE_DBG_INIT, "%s: max_mtu = %d\n",
			  prefix, caps->max_mtu);
		break;
	case ICE_AQC_CAPS_PCIE_RESET_AVOIDANCE:
		caps->pcie_reset_avoidance = (number > 0);
		DNPRINTF(ICE_DBG_INIT,
			  "%s: pcie_reset_avoidance = %d\n", prefix,
			  caps->pcie_reset_avoidance);
		break;
	case ICE_AQC_CAPS_POST_UPDATE_RESET_RESTRICT:
		caps->reset_restrict_support = (number == 1);
		DNPRINTF(ICE_DBG_INIT,
			  "%s: reset_restrict_support = %d\n", prefix,
			  caps->reset_restrict_support);
		break;
	case ICE_AQC_CAPS_EXT_TOPO_DEV_IMG0:
	case ICE_AQC_CAPS_EXT_TOPO_DEV_IMG1:
	case ICE_AQC_CAPS_EXT_TOPO_DEV_IMG2:
	case ICE_AQC_CAPS_EXT_TOPO_DEV_IMG3:
	{
		uint8_t index = (uint8_t)(cap - ICE_AQC_CAPS_EXT_TOPO_DEV_IMG0);

		caps->ext_topo_dev_img_ver_high[index] = number;
		caps->ext_topo_dev_img_ver_low[index] = logical_id;
		caps->ext_topo_dev_img_part_num[index] =
			(phys_id & ICE_EXT_TOPO_DEV_IMG_PART_NUM_M) >>
			ICE_EXT_TOPO_DEV_IMG_PART_NUM_S;
		caps->ext_topo_dev_img_load_en[index] =
			(phys_id & ICE_EXT_TOPO_DEV_IMG_LOAD_EN) != 0;
		caps->ext_topo_dev_img_prog_en[index] =
			(phys_id & ICE_EXT_TOPO_DEV_IMG_PROG_EN) != 0;
		caps->ext_topo_dev_img_ver_schema[index] =
			(phys_id & ICE_EXT_TOPO_DEV_IMG_VER_SCHEMA) != 0;
		DNPRINTF(ICE_DBG_INIT,
			  "%s: ext_topo_dev_img_ver_high[%d] = %d\n",
			  prefix, index,
			  caps->ext_topo_dev_img_ver_high[index]);
		DNPRINTF(ICE_DBG_INIT,
			  "%s: ext_topo_dev_img_ver_low[%d] = %d\n",
			  prefix, index,
			  caps->ext_topo_dev_img_ver_low[index]);
		DNPRINTF(ICE_DBG_INIT,
			  "%s: ext_topo_dev_img_part_num[%d] = %d\n",
			  prefix, index,
			  caps->ext_topo_dev_img_part_num[index]);
		DNPRINTF(ICE_DBG_INIT,
			  "%s: ext_topo_dev_img_load_en[%d] = %d\n",
			  prefix, index,
			  caps->ext_topo_dev_img_load_en[index]);
		DNPRINTF(ICE_DBG_INIT,
			  "%s: ext_topo_dev_img_prog_en[%d] = %d\n",
			  prefix, index,
			  caps->ext_topo_dev_img_prog_en[index]);
		DNPRINTF(ICE_DBG_INIT,
			  "%s: ext_topo_dev_img_ver_schema[%d] = %d\n",
			  prefix, index,
			  caps->ext_topo_dev_img_ver_schema[index]);
		break;
	}
	case ICE_AQC_CAPS_TX_SCHED_TOPO_COMP_MODE:
		caps->tx_sched_topo_comp_mode_en = (number == 1);
		break;
	case ICE_AQC_CAPS_DYN_FLATTENING:
		caps->dyn_flattening_en = (number == 1);
		DNPRINTF(ICE_DBG_INIT, "%s: dyn_flattening_en = %d\n",
			  prefix, caps->dyn_flattening_en);
		break;
	case ICE_AQC_CAPS_OROM_RECOVERY_UPDATE:
		caps->orom_recovery_update = (number == 1);
		DNPRINTF(ICE_DBG_INIT, "%s: orom_recovery_update = %d\n",
			  prefix, caps->orom_recovery_update);
		break;
	default:
		/* Not one of the recognized common capabilities */
		found = false;
	}

	return found;
}

/**
 * ice_recalc_port_limited_caps - Recalculate port limited capabilities
 * @hw: pointer to the HW structure
 * @caps: pointer to capabilities structure to fix
 *
 * Re-calculate the capabilities that are dependent on the number of physical
 * ports; i.e. some features are not supported or function differently on
 * devices with more than 4 ports.
 */
void
ice_recalc_port_limited_caps(struct ice_hw *hw, struct ice_hw_common_caps *caps)
{
	/* This assumes device capabilities are always scanned before function
	 * capabilities during the initialization flow.
	 */
	if (hw->dev_caps.num_funcs > 4) {
		/* Max 4 TCs per port */
		caps->maxtc = 4;
		DNPRINTF(ICE_DBG_INIT,
		    "reducing maxtc to %d (based on #ports)\n", caps->maxtc);
		if (caps->iwarp) {
			DNPRINTF(ICE_DBG_INIT, "forcing RDMA off\n");
			caps->iwarp = 0;
		}

		/* print message only when processing device capabilities
		 * during initialization.
		 */
		if (caps == &hw->dev_caps.common_cap)
			DPRINTF("RDMA functionality is not available with "
			    "the current device configuration.\n");
	}
}

/**
 * ice_parse_vf_func_caps - Parse ICE_AQC_CAPS_VF function caps
 * @hw: pointer to the HW struct
 * @func_p: pointer to function capabilities structure
 * @cap: pointer to the capability element to parse
 *
 * Extract function capabilities for ICE_AQC_CAPS_VF.
 */
void
ice_parse_vf_func_caps(struct ice_hw *hw, struct ice_hw_func_caps *func_p,
		       struct ice_aqc_list_caps_elem *cap)
{
	uint32_t number = le32toh(cap->number);
	uint32_t logical_id = le32toh(cap->logical_id);

	func_p->num_allocd_vfs = number;
	func_p->vf_base_id = logical_id;
	DNPRINTF(ICE_DBG_INIT, "func caps: num_allocd_vfs = %d\n",
		  func_p->num_allocd_vfs);
	DNPRINTF(ICE_DBG_INIT, "func caps: vf_base_id = %d\n",
		  func_p->vf_base_id);
}

/**
 * ice_parse_vsi_func_caps - Parse ICE_AQC_CAPS_VSI function caps
 * @hw: pointer to the HW struct
 * @func_p: pointer to function capabilities structure
 * @cap: pointer to the capability element to parse
 *
 * Extract function capabilities for ICE_AQC_CAPS_VSI.
 */
void
ice_parse_vsi_func_caps(struct ice_hw *hw, struct ice_hw_func_caps *func_p,
			struct ice_aqc_list_caps_elem *cap)
{
	func_p->guar_num_vsi = ice_get_num_per_func(hw, ICE_MAX_VSI);
	DNPRINTF(ICE_DBG_INIT, "func caps: guar_num_vsi (fw) = %d\n",
	    le32toh(cap->number));
	DNPRINTF(ICE_DBG_INIT, "func caps: guar_num_vsi = %d\n",
		  func_p->guar_num_vsi);
}

/**
 * ice_parse_func_caps - Parse function capabilities
 * @hw: pointer to the HW struct
 * @func_p: pointer to function capabilities structure
 * @buf: buffer containing the function capability records
 * @cap_count: the number of capabilities
 *
 * Helper function to parse function (0x000A) capabilities list. For
 * capabilities shared between device and function, this relies on
 * ice_parse_common_caps.
 *
 * Loop through the list of provided capabilities and extract the relevant
 * data into the function capabilities structured.
 */
void
ice_parse_func_caps(struct ice_hw *hw, struct ice_hw_func_caps *func_p,
		    void *buf, uint32_t cap_count)
{
	struct ice_aqc_list_caps_elem *cap_resp;
	uint32_t i;

	cap_resp = (struct ice_aqc_list_caps_elem *)buf;

	memset(func_p, 0, sizeof(*func_p));

	for (i = 0; i < cap_count; i++) {
		uint16_t cap = le16toh(cap_resp[i].cap);
		bool found;

		found = ice_parse_common_caps(hw, &func_p->common_cap,
					      &cap_resp[i], "func caps");

		switch (cap) {
		case ICE_AQC_CAPS_VF:
			ice_parse_vf_func_caps(hw, func_p, &cap_resp[i]);
			break;
		case ICE_AQC_CAPS_VSI:
			ice_parse_vsi_func_caps(hw, func_p, &cap_resp[i]);
			break;
		default:
			/* Don't list common capabilities as unknown */
			if (!found)
				DNPRINTF(ICE_DBG_INIT, "func caps: unknown "
				    "capability[%d]: 0x%x\n", i, cap);
			break;
		}
	}

	ice_print_led_caps(hw, &func_p->common_cap, "func caps", true);
	ice_print_sdp_caps(hw, &func_p->common_cap, "func caps", true);

	ice_recalc_port_limited_caps(hw, &func_p->common_cap);
}

/**
 * ice_parse_valid_functions_cap - Parse ICE_AQC_CAPS_VALID_FUNCTIONS caps
 * @hw: pointer to the HW struct
 * @dev_p: pointer to device capabilities structure
 * @cap: capability element to parse
 *
 * Parse ICE_AQC_CAPS_VALID_FUNCTIONS for device capabilities.
 */
void
ice_parse_valid_functions_cap(struct ice_hw *hw, struct ice_hw_dev_caps *dev_p,
			      struct ice_aqc_list_caps_elem *cap)
{
	uint32_t number = le32toh(cap->number);

	dev_p->num_funcs = ice_popcount32(number);
	DNPRINTF(ICE_DBG_INIT, "dev caps: num_funcs = %d\n",
		  dev_p->num_funcs);

}

/**
 * ice_parse_vf_dev_caps - Parse ICE_AQC_CAPS_VF device caps
 * @hw: pointer to the HW struct
 * @dev_p: pointer to device capabilities structure
 * @cap: capability element to parse
 *
 * Parse ICE_AQC_CAPS_VF for device capabilities.
 */
void
ice_parse_vf_dev_caps(struct ice_hw *hw, struct ice_hw_dev_caps *dev_p,
		      struct ice_aqc_list_caps_elem *cap)
{
	uint32_t number = le32toh(cap->number);

	dev_p->num_vfs_exposed = number;
	DNPRINTF(ICE_DBG_INIT, "dev_caps: num_vfs_exposed = %d\n",
		  dev_p->num_vfs_exposed);
}

/**
 * ice_parse_vsi_dev_caps - Parse ICE_AQC_CAPS_VSI device caps
 * @hw: pointer to the HW struct
 * @dev_p: pointer to device capabilities structure
 * @cap: capability element to parse
 *
 * Parse ICE_AQC_CAPS_VSI for device capabilities.
 */
void
ice_parse_vsi_dev_caps(struct ice_hw *hw, struct ice_hw_dev_caps *dev_p,
		       struct ice_aqc_list_caps_elem *cap)
{
	uint32_t number = le32toh(cap->number);

	dev_p->num_vsi_allocd_to_host = number;
	DNPRINTF(ICE_DBG_INIT, "dev caps: num_vsi_allocd_to_host = %d\n",
		  dev_p->num_vsi_allocd_to_host);
}

/**
 * ice_parse_nac_topo_dev_caps - Parse ICE_AQC_CAPS_NAC_TOPOLOGY cap
 * @hw: pointer to the HW struct
 * @dev_p: pointer to device capabilities structure
 * @cap: capability element to parse
 *
 * Parse ICE_AQC_CAPS_NAC_TOPOLOGY for device capabilities.
 */
void
ice_parse_nac_topo_dev_caps(struct ice_hw *hw, struct ice_hw_dev_caps *dev_p,
			    struct ice_aqc_list_caps_elem *cap)
{
	dev_p->nac_topo.mode = le32toh(cap->number);
	dev_p->nac_topo.id = le32toh(cap->phys_id) & ICE_NAC_TOPO_ID_M;

	DPRINTF("PF is configured in %s mode with IP instance ID %d\n",
		 (dev_p->nac_topo.mode == 0) ? "primary" : "secondary",
		 dev_p->nac_topo.id);

	DNPRINTF(ICE_DBG_INIT, "dev caps: nac topology is_primary = %d\n",
		  !!(dev_p->nac_topo.mode & ICE_NAC_TOPO_PRIMARY_M));
	DNPRINTF(ICE_DBG_INIT, "dev caps: nac topology is_dual = %d\n",
		  !!(dev_p->nac_topo.mode & ICE_NAC_TOPO_DUAL_M));
	DNPRINTF(ICE_DBG_INIT, "dev caps: nac topology id = %d\n",
		  dev_p->nac_topo.id);
}

/**
 * ice_is_vsi_valid - check whether the VSI is valid or not
 * @hw: pointer to the HW struct
 * @vsi_handle: VSI handle
 *
 * check whether the VSI is valid or not
 */
bool
ice_is_vsi_valid(struct ice_hw *hw, uint16_t vsi_handle)
{
	return vsi_handle < ICE_MAX_VSI && hw->vsi_ctx[vsi_handle];
}

/**
 * ice_parse_sensor_reading_cap - Parse ICE_AQC_CAPS_SENSOR_READING cap
 * @hw: pointer to the HW struct
 * @dev_p: pointer to device capabilities structure
 * @cap: capability element to parse
 *
 * Parse ICE_AQC_CAPS_SENSOR_READING for device capability for reading
 * enabled sensors.
 */
void
ice_parse_sensor_reading_cap(struct ice_hw *hw, struct ice_hw_dev_caps *dev_p,
			     struct ice_aqc_list_caps_elem *cap)
{
	dev_p->supported_sensors = le32toh(cap->number);

	DNPRINTF(ICE_DBG_INIT,
		  "dev caps: supported sensors (bitmap) = 0x%x\n",
		  dev_p->supported_sensors);
}

/**
 * ice_parse_dev_caps - Parse device capabilities
 * @hw: pointer to the HW struct
 * @dev_p: pointer to device capabilities structure
 * @buf: buffer containing the device capability records
 * @cap_count: the number of capabilities
 *
 * Helper device to parse device (0x000B) capabilities list. For
 * capabilities shared between device and function, this relies on
 * ice_parse_common_caps.
 *
 * Loop through the list of provided capabilities and extract the relevant
 * data into the device capabilities structured.
 */
void
ice_parse_dev_caps(struct ice_hw *hw, struct ice_hw_dev_caps *dev_p,
		   void *buf, uint32_t cap_count)
{
	struct ice_aqc_list_caps_elem *cap_resp;
	uint32_t i;

	cap_resp = (struct ice_aqc_list_caps_elem *)buf;

	memset(dev_p, 0, sizeof(*dev_p));

	for (i = 0; i < cap_count; i++) {
		uint16_t cap = le16toh(cap_resp[i].cap);
		bool found;

		found = ice_parse_common_caps(hw, &dev_p->common_cap,
					      &cap_resp[i], "dev caps");

		switch (cap) {
		case ICE_AQC_CAPS_VALID_FUNCTIONS:
			ice_parse_valid_functions_cap(hw, dev_p, &cap_resp[i]);
			break;
		case ICE_AQC_CAPS_VF:
			ice_parse_vf_dev_caps(hw, dev_p, &cap_resp[i]);
			break;
		case ICE_AQC_CAPS_VSI:
			ice_parse_vsi_dev_caps(hw, dev_p, &cap_resp[i]);
			break;
		case ICE_AQC_CAPS_NAC_TOPOLOGY:
			ice_parse_nac_topo_dev_caps(hw, dev_p, &cap_resp[i]);
			break;
		case ICE_AQC_CAPS_SENSOR_READING:
			ice_parse_sensor_reading_cap(hw, dev_p, &cap_resp[i]);
			break;
		default:
			/* Don't list common capabilities as unknown */
			if (!found)
				DNPRINTF(ICE_DBG_INIT,
				    "dev caps: unknown capability[%d]: 0x%x\n",
				    i, cap);
			break;
		}
	}

	ice_print_led_caps(hw, &dev_p->common_cap, "dev caps", true);
	ice_print_sdp_caps(hw, &dev_p->common_cap, "dev caps", true);

	ice_recalc_port_limited_caps(hw, &dev_p->common_cap);
}

/**
 * ice_aq_list_caps - query function/device capabilities
 * @hw: pointer to the HW struct
 * @buf: a buffer to hold the capabilities
 * @buf_size: size of the buffer
 * @cap_count: if not NULL, set to the number of capabilities reported
 * @opc: capabilities type to discover, device or function
 * @cd: pointer to command details structure or NULL
 *
 * Get the function (0x000A) or device (0x000B) capabilities description from
 * firmware and store it in the buffer.
 *
 * If the cap_count pointer is not NULL, then it is set to the number of
 * capabilities firmware will report. Note that if the buffer size is too
 * small, it is possible the command will return ICE_AQ_ERR_ENOMEM. The
 * cap_count will still be updated in this case. It is recommended that the
 * buffer size be set to ICE_AQ_MAX_BUF_LEN (the largest possible buffer that
 * firmware could return) to avoid this.
 */
enum ice_status
ice_aq_list_caps(struct ice_hw *hw, void *buf, uint16_t buf_size,
    uint32_t *cap_count, enum ice_adminq_opc opc, struct ice_sq_cd *cd)
{
	struct ice_aqc_list_caps *cmd;
	struct ice_aq_desc desc;
	enum ice_status status;

	cmd = &desc.params.get_cap;

	if (opc != ice_aqc_opc_list_func_caps &&
	    opc != ice_aqc_opc_list_dev_caps)
		return ICE_ERR_PARAM;

	ice_fill_dflt_direct_cmd_desc(&desc, opc);
	status = ice_aq_send_cmd(hw, &desc, buf, buf_size, cd);

	if (cap_count)
		*cap_count = le32toh(cmd->count);

	return status;
}

/**
 * ice_discover_dev_caps - Read and extract device capabilities
 * @hw: pointer to the hardware structure
 * @dev_caps: pointer to device capabilities structure
 *
 * Read the device capabilities and extract them into the dev_caps structure
 * for later use.
 */
enum ice_status
ice_discover_dev_caps(struct ice_hw *hw, struct ice_hw_dev_caps *dev_caps)
{
	enum ice_status status;
	uint32_t cap_count = 0;
	void *cbuf;

	cbuf = ice_malloc(hw, ICE_AQ_MAX_BUF_LEN);
	if (!cbuf)
		return ICE_ERR_NO_MEMORY;

	/* Although the driver doesn't know the number of capabilities the
	 * device will return, we can simply send a 4KB buffer, the maximum
	 * possible size that firmware can return.
	 */
	cap_count = ICE_AQ_MAX_BUF_LEN / sizeof(struct ice_aqc_list_caps_elem);

	status = ice_aq_list_caps(hw, cbuf, ICE_AQ_MAX_BUF_LEN, &cap_count,
				  ice_aqc_opc_list_dev_caps, NULL);
	if (!status)
		ice_parse_dev_caps(hw, dev_caps, cbuf, cap_count);
	ice_free(hw, cbuf);

	return status;
}

/**
 * ice_discover_func_caps - Read and extract function capabilities
 * @hw: pointer to the hardware structure
 * @func_caps: pointer to function capabilities structure
 *
 * Read the function capabilities and extract them into the func_caps structure
 * for later use.
 */
enum ice_status
ice_discover_func_caps(struct ice_hw *hw, struct ice_hw_func_caps *func_caps)
{
	enum ice_status status;
	uint32_t cap_count = 0;
	void *cbuf;

	cbuf = ice_malloc(hw, ICE_AQ_MAX_BUF_LEN);
	if (!cbuf)
		return ICE_ERR_NO_MEMORY;

	/* Although the driver doesn't know the number of capabilities the
	 * device will return, we can simply send a 4KB buffer, the maximum
	 * possible size that firmware can return.
	 */
	cap_count = ICE_AQ_MAX_BUF_LEN / sizeof(struct ice_aqc_list_caps_elem);

	status = ice_aq_list_caps(hw, cbuf, ICE_AQ_MAX_BUF_LEN, &cap_count,
				  ice_aqc_opc_list_func_caps, NULL);
	if (!status)
		ice_parse_func_caps(hw, func_caps, cbuf, cap_count);
	ice_free(hw, cbuf);

	return status;
}

/**
 * ice_set_safe_mode_caps - Override dev/func capabilities when in safe mode
 * @hw: pointer to the hardware structure
 */
void
ice_set_safe_mode_caps(struct ice_hw *hw)
{
	struct ice_hw_func_caps *func_caps = &hw->func_caps;
	struct ice_hw_dev_caps *dev_caps = &hw->dev_caps;
	struct ice_hw_common_caps cached_caps;
	uint32_t num_funcs;

	/* cache some func_caps values that should be restored after memset */
	cached_caps = func_caps->common_cap;

	/* unset func capabilities */
	memset(func_caps, 0, sizeof(*func_caps));

#define ICE_RESTORE_FUNC_CAP(name) \
	func_caps->common_cap.name = cached_caps.name

	/* restore cached values */
	ICE_RESTORE_FUNC_CAP(valid_functions);
	ICE_RESTORE_FUNC_CAP(txq_first_id);
	ICE_RESTORE_FUNC_CAP(rxq_first_id);
	ICE_RESTORE_FUNC_CAP(msix_vector_first_id);
	ICE_RESTORE_FUNC_CAP(max_mtu);
	ICE_RESTORE_FUNC_CAP(nvm_unified_update);

	/* one Tx and one Rx queue in safe mode */
	func_caps->common_cap.num_rxq = 1;
	func_caps->common_cap.num_txq = 1;

	/* two MSIX vectors, one for traffic and one for misc causes */
	func_caps->common_cap.num_msix_vectors = 2;
	func_caps->guar_num_vsi = 1;

	/* cache some dev_caps values that should be restored after memset */
	cached_caps = dev_caps->common_cap;
	num_funcs = dev_caps->num_funcs;

	/* unset dev capabilities */
	memset(dev_caps, 0, sizeof(*dev_caps));

#define ICE_RESTORE_DEV_CAP(name) \
	dev_caps->common_cap.name = cached_caps.name

	/* restore cached values */
	ICE_RESTORE_DEV_CAP(valid_functions);
	ICE_RESTORE_DEV_CAP(txq_first_id);
	ICE_RESTORE_DEV_CAP(rxq_first_id);
	ICE_RESTORE_DEV_CAP(msix_vector_first_id);
	ICE_RESTORE_DEV_CAP(max_mtu);
	ICE_RESTORE_DEV_CAP(nvm_unified_update);
	dev_caps->num_funcs = num_funcs;

	/* one Tx and one Rx queue per function in safe mode */
	dev_caps->common_cap.num_rxq = num_funcs;
	dev_caps->common_cap.num_txq = num_funcs;

	/* two MSIX vectors per function */
	dev_caps->common_cap.num_msix_vectors = 2 * num_funcs;
}

/**
 * ice_get_caps - get info about the HW
 * @hw: pointer to the hardware structure
 */
enum ice_status
ice_get_caps(struct ice_hw *hw)
{
	enum ice_status status;

	status = ice_discover_dev_caps(hw, &hw->dev_caps);
	if (status)
		return status;

	return ice_discover_func_caps(hw, &hw->func_caps);
}

/**
 * ice_aq_get_sw_cfg - get switch configuration
 * @hw: pointer to the hardware structure
 * @buf: pointer to the result buffer
 * @buf_size: length of the buffer available for response
 * @req_desc: pointer to requested descriptor
 * @num_elems: pointer to number of elements
 * @cd: pointer to command details structure or NULL
 *
 * Get switch configuration (0x0200) to be placed in buf.
 * This admin command returns information such as initial VSI/port number
 * and switch ID it belongs to.
 *
 * NOTE: *req_desc is both an input/output parameter.
 * The caller of this function first calls this function with *request_desc set
 * to 0. If the response from f/w has *req_desc set to 0, all the switch
 * configuration information has been returned; if non-zero (meaning not all
 * the information was returned), the caller should call this function again
 * with *req_desc set to the previous value returned by f/w to get the
 * next block of switch configuration information.
 *
 * *num_elems is output only parameter. This reflects the number of elements
 * in response buffer. The caller of this function to use *num_elems while
 * parsing the response buffer.
 */
enum ice_status
ice_aq_get_sw_cfg(struct ice_hw *hw, struct ice_aqc_get_sw_cfg_resp_elem *buf,
		  uint16_t buf_size, uint16_t *req_desc, uint16_t *num_elems,
		  struct ice_sq_cd *cd)
{
	struct ice_aqc_get_sw_cfg *cmd;
	struct ice_aq_desc desc;
	enum ice_status status;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_get_sw_cfg);
	cmd = &desc.params.get_sw_conf;
	cmd->element = htole16(*req_desc);

	status = ice_aq_send_cmd(hw, &desc, buf, buf_size, cd);
	if (!status) {
		*req_desc = le16toh(cmd->element);
		*num_elems = le16toh(cmd->num_elems);
	}

	return status;
}

/* ice_init_port_info - Initialize port_info with switch configuration data
 * @pi: pointer to port_info
 * @vsi_port_num: VSI number or port number
 * @type: Type of switch element (port or VSI)
 * @swid: switch ID of the switch the element is attached to
 * @pf_vf_num: PF or VF number
 * @is_vf: true if the element is a VF, false otherwise
 */
void
ice_init_port_info(struct ice_port_info *pi, uint16_t vsi_port_num,
    uint8_t type, uint16_t swid, uint16_t pf_vf_num, bool is_vf)
{
	switch (type) {
	case ICE_AQC_GET_SW_CONF_RESP_PHYS_PORT:
		pi->lport = (uint8_t)(vsi_port_num & ICE_LPORT_MASK);
		pi->sw_id = swid;
		pi->pf_vf_num = pf_vf_num;
		pi->is_vf = is_vf;
		break;
	default:
		DNPRINTF(ICE_DBG_SW, "incorrect VSI/port type received\n");
		break;
	}
}

/* ice_get_initial_sw_cfg - Get initial port and default VSI data */
enum ice_status
ice_get_initial_sw_cfg(struct ice_hw *hw)
{
	struct ice_aqc_get_sw_cfg_resp_elem *rbuf;
	enum ice_status status;
	uint8_t num_total_ports;
	uint16_t req_desc = 0;
	uint16_t num_elems;
	uint8_t j = 0;
	uint16_t i;

	num_total_ports = 1;

	rbuf = (struct ice_aqc_get_sw_cfg_resp_elem *)
		ice_malloc(hw, ICE_SW_CFG_MAX_BUF_LEN);

	if (!rbuf)
		return ICE_ERR_NO_MEMORY;

	/* Multiple calls to ice_aq_get_sw_cfg may be required
	 * to get all the switch configuration information. The need
	 * for additional calls is indicated by ice_aq_get_sw_cfg
	 * writing a non-zero value in req_desc
	 */
	do {
		struct ice_aqc_get_sw_cfg_resp_elem *ele;

		status = ice_aq_get_sw_cfg(hw, rbuf, ICE_SW_CFG_MAX_BUF_LEN,
					   &req_desc, &num_elems, NULL);

		if (status)
			break;

		for (i = 0, ele = rbuf; i < num_elems; i++, ele++) {
			uint16_t pf_vf_num, swid, vsi_port_num;
			bool is_vf = false;
			uint8_t res_type;

			vsi_port_num = le16toh(ele->vsi_port_num) &
				ICE_AQC_GET_SW_CONF_RESP_VSI_PORT_NUM_M;

			pf_vf_num = le16toh(ele->pf_vf_num) &
				ICE_AQC_GET_SW_CONF_RESP_FUNC_NUM_M;

			swid = le16toh(ele->swid);

			if (le16toh(ele->pf_vf_num) &
			    ICE_AQC_GET_SW_CONF_RESP_IS_VF)
				is_vf = true;

			res_type = (uint8_t)(le16toh(ele->vsi_port_num) >>
					ICE_AQC_GET_SW_CONF_RESP_TYPE_S);

			switch (res_type) {
			case ICE_AQC_GET_SW_CONF_RESP_PHYS_PORT:
			case ICE_AQC_GET_SW_CONF_RESP_VIRT_PORT:
				if (j == num_total_ports) {
					DNPRINTF(ICE_DBG_SW,
					    "more ports than expected\n");
					status = ICE_ERR_CFG;
					goto out;
				}
				ice_init_port_info(hw->port_info,
						   vsi_port_num, res_type, swid,
						   pf_vf_num, is_vf);
				j++;
				break;
			default:
				break;
			}
		}
	} while (req_desc && !status);

out:
	ice_free(hw, rbuf);
	return status;
}

/**
 * ice_aq_query_sched_res - query scheduler resource
 * @hw: pointer to the HW struct
 * @buf_size: buffer size in bytes
 * @buf: pointer to buffer
 * @cd: pointer to command details structure or NULL
 *
 * Query scheduler resource allocation (0x0412)
 */
enum ice_status
ice_aq_query_sched_res(struct ice_hw *hw, uint16_t buf_size,
		       struct ice_aqc_query_txsched_res_resp *buf,
		       struct ice_sq_cd *cd)
{
	struct ice_aq_desc desc;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_query_sched_res);
	return ice_aq_send_cmd(hw, &desc, buf, buf_size, cd);
}

/**
 * ice_sched_query_res_alloc - query the FW for num of logical sched layers
 */
enum ice_status
ice_sched_query_res_alloc(struct ice_hw *hw)
{
	struct ice_aqc_query_txsched_res_resp *buf;
	enum ice_status status = ICE_SUCCESS;
	uint16_t max_sibl;
	uint8_t i;

	if (hw->layer_info)
		return status;

	buf = (struct ice_aqc_query_txsched_res_resp *)
		ice_malloc(hw, sizeof(*buf));
	if (!buf)
		return ICE_ERR_NO_MEMORY;

	status = ice_aq_query_sched_res(hw, sizeof(*buf), buf, NULL);
	if (status)
		goto sched_query_out;

	hw->num_tx_sched_layers =
		(uint8_t)le16toh(buf->sched_props.logical_levels);
	hw->num_tx_sched_phys_layers =
		(uint8_t)le16toh(buf->sched_props.phys_levels);
	hw->flattened_layers = buf->sched_props.flattening_bitmap;
	hw->max_cgds = buf->sched_props.max_pf_cgds;

	/* max sibling group size of current layer refers to the max children
	 * of the below layer node.
	 * layer 1 node max children will be layer 2 max sibling group size
	 * layer 2 node max children will be layer 3 max sibling group size
	 * and so on. This array will be populated from root (index 0) to
	 * qgroup layer 7. Leaf node has no children.
	 */
	for (i = 0; i < hw->num_tx_sched_layers - 1; i++) {
		max_sibl = buf->layer_props[i + 1].max_sibl_grp_sz;
		hw->max_children[i] = le16toh(max_sibl);
	}

	hw->layer_info = (struct ice_aqc_layer_props *)
			 ice_memdup(hw, buf->layer_props,
				    (hw->num_tx_sched_layers *
				     sizeof(*hw->layer_info)));
	if (!hw->layer_info) {
		status = ICE_ERR_NO_MEMORY;
		goto sched_query_out;
	}

sched_query_out:
	ice_free(hw, buf);
	return status;
}

/**
 * ice_sched_get_psm_clk_freq - determine the PSM clock frequency
 * @hw: pointer to the HW struct
 *
 * Determine the PSM clock frequency and store in HW struct
 */
void
ice_sched_get_psm_clk_freq(struct ice_hw *hw)
{
	uint32_t val, clk_src;

	val = ICE_READ(hw, GLGEN_CLKSTAT_SRC);
	clk_src = (val & GLGEN_CLKSTAT_SRC_PSM_CLK_SRC_M) >>
		GLGEN_CLKSTAT_SRC_PSM_CLK_SRC_S;

	switch (clk_src) {
	case PSM_CLK_SRC_367_MHZ:
		hw->psm_clk_freq = ICE_PSM_CLK_367MHZ_IN_HZ;
		break;
	case PSM_CLK_SRC_416_MHZ:
		hw->psm_clk_freq = ICE_PSM_CLK_416MHZ_IN_HZ;
		break;
	case PSM_CLK_SRC_446_MHZ:
		hw->psm_clk_freq = ICE_PSM_CLK_446MHZ_IN_HZ;
		break;
	case PSM_CLK_SRC_390_MHZ:
		hw->psm_clk_freq = ICE_PSM_CLK_390MHZ_IN_HZ;
		break;

	/* default condition is not required as clk_src is restricted
	 * to a 2-bit value from GLGEN_CLKSTAT_SRC_PSM_CLK_SRC_M mask.
	 * The above switch statements cover the possible values of
	 * this variable.
	 */
	}
}

/**
 * ice_aq_get_dflt_topo - gets default scheduler topology
 * @hw: pointer to the HW struct
 * @lport: logical port number
 * @buf: pointer to buffer
 * @buf_size: buffer size in bytes
 * @num_branches: returns total number of queue to port branches
 * @cd: pointer to command details structure or NULL
 *
 * Get default scheduler topology (0x400)
 */
enum ice_status
ice_aq_get_dflt_topo(struct ice_hw *hw, uint8_t lport,
		     struct ice_aqc_get_topo_elem *buf, uint16_t buf_size,
		     uint8_t *num_branches, struct ice_sq_cd *cd)
{
	struct ice_aqc_get_topo *cmd;
	struct ice_aq_desc desc;
	enum ice_status status;

	cmd = &desc.params.get_topo;
	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_get_dflt_topo);
	cmd->port_num = lport;
	status = ice_aq_send_cmd(hw, &desc, buf, buf_size, cd);
	if (!status && num_branches)
		*num_branches = cmd->num_branches;

	return status;
}

/**
 * ice_sched_add_root_node - Insert the Tx scheduler root node in SW DB
 * @pi: port information structure
 * @info: Scheduler element information from firmware
 *
 * This function inserts the root node of the scheduling tree topology
 * to the SW DB.
 */
enum ice_status
ice_sched_add_root_node(struct ice_port_info *pi,
			struct ice_aqc_txsched_elem_data *info)
{
	struct ice_sched_node *root;
	struct ice_hw *hw;

	if (!pi)
		return ICE_ERR_PARAM;

	hw = pi->hw;

	root = (struct ice_sched_node *)ice_malloc(hw, sizeof(*root));
	if (!root)
		return ICE_ERR_NO_MEMORY;

	root->children = (struct ice_sched_node **)
		ice_calloc(hw, hw->max_children[0], sizeof(*root->children));
	if (!root->children) {
		ice_free(hw, root);
		return ICE_ERR_NO_MEMORY;
	}

	memcpy(&root->info, info, sizeof(*info));
	pi->root = root;
	return ICE_SUCCESS;
}

/**
 * ice_sched_find_node_by_teid - Find the Tx scheduler node in SW DB
 * @start_node: pointer to the starting ice_sched_node struct in a sub-tree
 * @teid: node TEID to search
 *
 * This function searches for a node matching the TEID in the scheduling tree
 * from the SW DB. The search is recursive and is restricted by the number of
 * layers it has searched through; stopping at the max supported layer.
 *
 * This function needs to be called when holding the port_info->sched_lock
 */
struct ice_sched_node *
ice_sched_find_node_by_teid(struct ice_sched_node *start_node, uint32_t teid)
{
	uint16_t i;

	/* The TEID is same as that of the start_node */
	if (ICE_TXSCHED_GET_NODE_TEID(start_node) == teid)
		return start_node;

	/* The node has no children or is at the max layer */
	if (!start_node->num_children ||
	    start_node->tx_sched_layer >= ICE_AQC_TOPO_MAX_LEVEL_NUM ||
	    start_node->info.data.elem_type == ICE_AQC_ELEM_TYPE_LEAF)
		return NULL;

	/* Check if TEID matches to any of the children nodes */
	for (i = 0; i < start_node->num_children; i++)
		if (ICE_TXSCHED_GET_NODE_TEID(start_node->children[i]) == teid)
			return start_node->children[i];

	/* Search within each child's sub-tree */
	for (i = 0; i < start_node->num_children; i++) {
		struct ice_sched_node *tmp;

		tmp = ice_sched_find_node_by_teid(start_node->children[i],
						  teid);
		if (tmp)
			return tmp;
	}

	return NULL;
}

/**
 * ice_sched_get_tc_node - get pointer to TC node
 * @pi: port information structure
 * @tc: TC number
 *
 * This function returns the TC node pointer
 */
struct ice_sched_node *
ice_sched_get_tc_node(struct ice_port_info *pi, uint8_t tc)
{
	uint8_t i;

	if (!pi || !pi->root)
		return NULL;
	for (i = 0; i < pi->root->num_children; i++)
		if (pi->root->children[i]->tc_num == tc)
			return pi->root->children[i];
	return NULL;
}

/**
 * ice_aqc_send_sched_elem_cmd - send scheduling elements cmd
 * @hw: pointer to the HW struct
 * @cmd_opc: cmd opcode
 * @elems_req: number of elements to request
 * @buf: pointer to buffer
 * @buf_size: buffer size in bytes
 * @elems_resp: returns total number of elements response
 * @cd: pointer to command details structure or NULL
 *
 * This function sends a scheduling elements cmd (cmd_opc)
 */
enum ice_status
ice_aqc_send_sched_elem_cmd(struct ice_hw *hw, enum ice_adminq_opc cmd_opc,
			    uint16_t elems_req, void *buf, uint16_t buf_size,
			    uint16_t *elems_resp, struct ice_sq_cd *cd)
{
	struct ice_aqc_sched_elem_cmd *cmd;
	struct ice_aq_desc desc;
	enum ice_status status;

	cmd = &desc.params.sched_elem_cmd;
	ice_fill_dflt_direct_cmd_desc(&desc, cmd_opc);
	cmd->num_elem_req = htole16(elems_req);
	desc.flags |= htole16(ICE_AQ_FLAG_RD);
	status = ice_aq_send_cmd(hw, &desc, buf, buf_size, cd);
	if (!status && elems_resp)
		*elems_resp = le16toh(cmd->num_elem_resp);

	return status;
}

/**
 * ice_aq_query_sched_elems - query scheduler elements
 * @hw: pointer to the HW struct
 * @elems_req: number of elements to query
 * @buf: pointer to buffer
 * @buf_size: buffer size in bytes
 * @elems_ret: returns total number of elements returned
 * @cd: pointer to command details structure or NULL
 *
 * Query scheduling elements (0x0404)
 */
enum ice_status
ice_aq_query_sched_elems(struct ice_hw *hw, uint16_t elems_req,
    struct ice_aqc_txsched_elem_data *buf, uint16_t buf_size,
    uint16_t *elems_ret, struct ice_sq_cd *cd)
{
	return ice_aqc_send_sched_elem_cmd(hw, ice_aqc_opc_get_sched_elems,
					   elems_req, (void *)buf, buf_size,
					   elems_ret, cd);
}

/**
 * ice_sched_query_elem - query element information from HW
 * @hw: pointer to the HW struct
 * @node_teid: node TEID to be queried
 * @buf: buffer to element information
 *
 * This function queries HW element information
 */
enum ice_status
ice_sched_query_elem(struct ice_hw *hw, uint32_t node_teid,
		     struct ice_aqc_txsched_elem_data *buf)
{
	uint16_t buf_size, num_elem_ret = 0;
	enum ice_status status;

	buf_size = sizeof(*buf);
	memset(buf, 0, buf_size);
	buf->node_teid = htole32(node_teid);
	status = ice_aq_query_sched_elems(hw, 1, buf, buf_size, &num_elem_ret,
					  NULL);
	if (status != ICE_SUCCESS || num_elem_ret != 1)
		DNPRINTF(ICE_DBG_SCHED, "query element failed\n");
	return status;
}

/**
 * ice_sched_add_node - Insert the Tx scheduler node in SW DB
 * @pi: port information structure
 * @layer: Scheduler layer of the node
 * @info: Scheduler element information from firmware
 * @prealloc_node: preallocated ice_sched_node struct for SW DB
 *
 * This function inserts a scheduler node to the SW DB.
 */
enum ice_status
ice_sched_add_node(struct ice_port_info *pi, uint8_t layer,
		   struct ice_aqc_txsched_elem_data *info,
		   struct ice_sched_node *prealloc_node)
{
	struct ice_aqc_txsched_elem_data elem;
	struct ice_sched_node *parent;
	struct ice_sched_node *node;
	enum ice_status status;
	struct ice_hw *hw;

	if (!pi)
		return ICE_ERR_PARAM;

	hw = pi->hw;

	/* A valid parent node should be there */
	parent = ice_sched_find_node_by_teid(pi->root,
					     le32toh(info->parent_teid));
	if (!parent) {
		DNPRINTF(ICE_DBG_SCHED,
		    "Parent Node not found for parent_teid=0x%x\n",
		    le32toh(info->parent_teid));
		return ICE_ERR_PARAM;
	}

	/* query the current node information from FW before adding it
	 * to the SW DB
	 */
	status = ice_sched_query_elem(hw, le32toh(info->node_teid), &elem);
	if (status)
		return status;

	if (prealloc_node)
		node = prealloc_node;
	else
		node = (struct ice_sched_node *)ice_malloc(hw, sizeof(*node));
	if (!node)
		return ICE_ERR_NO_MEMORY;
	if (hw->max_children[layer]) {
		node->children = (struct ice_sched_node **)
			ice_calloc(hw, hw->max_children[layer],
				   sizeof(*node->children));
		if (!node->children) {
			ice_free(hw, node);
			return ICE_ERR_NO_MEMORY;
		}
	}

	node->in_use = true;
	node->parent = parent;
	node->tx_sched_layer = layer;
	parent->children[parent->num_children++] = node;
	node->info = elem;
	return ICE_SUCCESS;
}

/**
 * ice_aq_delete_sched_elems - delete scheduler elements
 * @hw: pointer to the HW struct
 * @grps_req: number of groups to delete
 * @buf: pointer to buffer
 * @buf_size: buffer size in bytes
 * @grps_del: returns total number of elements deleted
 * @cd: pointer to command details structure or NULL
 *
 * Delete scheduling elements (0x040F)
 */
enum ice_status
ice_aq_delete_sched_elems(struct ice_hw *hw, uint16_t grps_req,
    struct ice_aqc_delete_elem *buf, uint16_t buf_size, uint16_t *grps_del,
    struct ice_sq_cd *cd)
{
	return ice_aqc_send_sched_elem_cmd(hw, ice_aqc_opc_delete_sched_elems,
					   grps_req, (void *)buf, buf_size,
					   grps_del, cd);
}

/**
 * ice_sched_remove_elems - remove nodes from HW
 * @hw: pointer to the HW struct
 * @parent: pointer to the parent node
 * @num_nodes: number of nodes
 * @node_teids: array of node teids to be deleted
 *
 * This function remove nodes from HW
 */
enum ice_status
ice_sched_remove_elems(struct ice_hw *hw, struct ice_sched_node *parent,
		       uint16_t num_nodes, uint32_t *node_teids)
{
	struct ice_aqc_delete_elem *buf;
	uint16_t i, num_groups_removed = 0;
	enum ice_status status;
	uint16_t buf_size;

	buf_size = ice_struct_size(buf, teid, num_nodes);
	buf = (struct ice_aqc_delete_elem *)ice_malloc(hw, buf_size);
	if (!buf)
		return ICE_ERR_NO_MEMORY;

	buf->hdr.parent_teid = parent->info.node_teid;
	buf->hdr.num_elems = htole16(num_nodes);
	for (i = 0; i < num_nodes; i++)
		buf->teid[i] = htole32(node_teids[i]);

	status = ice_aq_delete_sched_elems(hw, 1, buf, buf_size,
					   &num_groups_removed, NULL);
	if (status != ICE_SUCCESS || num_groups_removed != 1)
		DNPRINTF(ICE_DBG_SCHED, "remove node failed FW error %d\n",
			  hw->adminq.sq_last_status);

	ice_free(hw, buf);
	return status;
}

/**
 * ice_sched_get_first_node - get the first node of the given layer
 * @pi: port information structure
 * @parent: pointer the base node of the subtree
 * @layer: layer number
 *
 * This function retrieves the first node of the given layer from the subtree
 */
struct ice_sched_node *
ice_sched_get_first_node(struct ice_port_info *pi,
			 struct ice_sched_node *parent, uint8_t layer)
{
	return pi->sib_head[parent->tc_num][layer];
}

/**
 * ice_free_sched_node - Free a Tx scheduler node from SW DB
 * @pi: port information structure
 * @node: pointer to the ice_sched_node struct
 *
 * This function frees up a node from SW DB as well as from HW
 *
 * This function needs to be called with the port_info->sched_lock held
 */
void
ice_free_sched_node(struct ice_port_info *pi, struct ice_sched_node *node)
{
	struct ice_sched_node *parent;
	struct ice_hw *hw = pi->hw;
	uint8_t i, j;

	/* Free the children before freeing up the parent node
	 * The parent array is updated below and that shifts the nodes
	 * in the array. So always pick the first child if num children > 0
	 */
	while (node->num_children)
		ice_free_sched_node(pi, node->children[0]);

	/* Leaf, TC and root nodes can't be deleted by SW */
	if (node->tx_sched_layer >= hw->sw_entry_point_layer &&
	    node->info.data.elem_type != ICE_AQC_ELEM_TYPE_TC &&
	    node->info.data.elem_type != ICE_AQC_ELEM_TYPE_ROOT_PORT &&
	    node->info.data.elem_type != ICE_AQC_ELEM_TYPE_LEAF) {
		uint32_t teid = le32toh(node->info.node_teid);

		ice_sched_remove_elems(hw, node->parent, 1, &teid);
	}
	parent = node->parent;
	/* root has no parent */
	if (parent) {
		struct ice_sched_node *p;

		/* update the parent */
		for (i = 0; i < parent->num_children; i++)
			if (parent->children[i] == node) {
				for (j = i + 1; j < parent->num_children; j++)
					parent->children[j - 1] =
						parent->children[j];
				parent->num_children--;
				break;
			}

		p = ice_sched_get_first_node(pi, node, node->tx_sched_layer);
		while (p) {
			if (p->sibling == node) {
				p->sibling = node->sibling;
				break;
			}
			p = p->sibling;
		}

		/* update the sibling head if head is getting removed */
		if (pi->sib_head[node->tc_num][node->tx_sched_layer] == node)
			pi->sib_head[node->tc_num][node->tx_sched_layer] =
				node->sibling;
	}

	/* leaf nodes have no children */
	if (node->children)
		ice_free(hw, node->children);
	ice_free(hw, node);
}

/**
 * ice_rm_dflt_leaf_node - remove the default leaf node in the tree
 * @pi: port information structure
 *
 * This function removes the leaf node that was created by the FW
 * during initialization
 */
void
ice_rm_dflt_leaf_node(struct ice_port_info *pi)
{
	struct ice_sched_node *node;

	node = pi->root;
	while (node) {
		if (!node->num_children)
			break;
		node = node->children[0];
	}
	if (node && node->info.data.elem_type == ICE_AQC_ELEM_TYPE_LEAF) {
		uint32_t teid = le32toh(node->info.node_teid);
		enum ice_status status;

		/* remove the default leaf node */
		status = ice_sched_remove_elems(pi->hw, node->parent, 1, &teid);
		if (!status)
			ice_free_sched_node(pi, node);
	}
}

/**
 * ice_sched_rm_dflt_nodes - free the default nodes in the tree
 * @pi: port information structure
 *
 * This function frees all the nodes except root and TC that were created by
 * the FW during initialization
 */
void
ice_sched_rm_dflt_nodes(struct ice_port_info *pi)
{
	struct ice_sched_node *node;

	ice_rm_dflt_leaf_node(pi);

	/* remove the default nodes except TC and root nodes */
	node = pi->root;
	while (node) {
		if (node->tx_sched_layer >= pi->hw->sw_entry_point_layer &&
		    node->info.data.elem_type != ICE_AQC_ELEM_TYPE_TC &&
		    node->info.data.elem_type != ICE_AQC_ELEM_TYPE_ROOT_PORT) {
			ice_free_sched_node(pi, node);
			break;
		}

		if (!node->num_children)
			break;
		node = node->children[0];
	}
}

/**
 * ice_sched_init_port - Initialize scheduler by querying information from FW
 * @pi: port info structure for the tree to cleanup
 *
 * This function is the initial call to find the total number of Tx scheduler
 * resources, default topology created by firmware and storing the information
 * in SW DB.
 */
enum ice_status
ice_sched_init_port(struct ice_port_info *pi)
{
	struct ice_aqc_get_topo_elem *buf;
	enum ice_status status;
	struct ice_hw *hw;
	uint8_t num_branches;
	uint16_t num_elems;
	uint8_t i, j;

	if (!pi)
		return ICE_ERR_PARAM;
	hw = pi->hw;

	/* Query the Default Topology from FW */
	buf = (struct ice_aqc_get_topo_elem *)ice_malloc(hw,
							 ICE_AQ_MAX_BUF_LEN);
	if (!buf)
		return ICE_ERR_NO_MEMORY;

	/* Query default scheduling tree topology */
	status = ice_aq_get_dflt_topo(hw, pi->lport, buf, ICE_AQ_MAX_BUF_LEN,
				      &num_branches, NULL);
	if (status)
		goto err_init_port;

	/* num_branches should be between 1-8 */
	if (num_branches < 1 || num_branches > ICE_TXSCHED_MAX_BRANCHES) {
		DNPRINTF(ICE_DBG_SCHED, "num_branches unexpected %d\n",
		    num_branches);
		status = ICE_ERR_PARAM;
		goto err_init_port;
	}

	/* get the number of elements on the default/first branch */
	num_elems = le16toh(buf[0].hdr.num_elems);

	/* num_elems should always be between 1-9 */
	if (num_elems < 1 || num_elems > ICE_AQC_TOPO_MAX_LEVEL_NUM) {
		DNPRINTF(ICE_DBG_SCHED, "num_elems unexpected %d\n", num_elems);
		status = ICE_ERR_PARAM;
		goto err_init_port;
	}

	/* If the last node is a leaf node then the index of the queue group
	 * layer is two less than the number of elements.
	 */
	if (num_elems > 2 && buf[0].generic[num_elems - 1].data.elem_type ==
	    ICE_AQC_ELEM_TYPE_LEAF)
		pi->last_node_teid =
			le32toh(buf[0].generic[num_elems - 2].node_teid);
	else
		pi->last_node_teid =
			le32toh(buf[0].generic[num_elems - 1].node_teid);

	/* Insert the Tx Sched root node */
	status = ice_sched_add_root_node(pi, &buf[0].generic[0]);
	if (status)
		goto err_init_port;

	/* Parse the default tree and cache the information */
	for (i = 0; i < num_branches; i++) {
		num_elems = le16toh(buf[i].hdr.num_elems);

		/* Skip root element as already inserted */
		for (j = 1; j < num_elems; j++) {
			/* update the sw entry point */
			if (buf[0].generic[j].data.elem_type ==
			    ICE_AQC_ELEM_TYPE_ENTRY_POINT)
				hw->sw_entry_point_layer = j;

			status = ice_sched_add_node(pi, j,
			    &buf[i].generic[j], NULL);
			if (status)
				goto err_init_port;
		}
	}

	/* Remove the default nodes. */
	if (pi->root)
		ice_sched_rm_dflt_nodes(pi);

	/* initialize the port for handling the scheduler tree */
	pi->port_state = ICE_SCHED_PORT_STATE_READY;
	ice_init_lock(&pi->sched_lock);
	for (i = 0; i < ICE_AQC_TOPO_MAX_LEVEL_NUM; i++)
		TAILQ_INIT(&hw->rl_prof_list[i]);

err_init_port:
	if (status && pi->root) {
		ice_free_sched_node(pi, pi->root);
		pi->root = NULL;
	}

	ice_free(hw, buf);
	return status;
}

/**
 * ice_aq_rl_profile - performs a rate limiting task
 * @hw: pointer to the HW struct
 * @opcode: opcode for add, query, or remove profile(s)
 * @num_profiles: the number of profiles
 * @buf: pointer to buffer
 * @buf_size: buffer size in bytes
 * @num_processed: number of processed add or remove profile(s) to return
 * @cd: pointer to command details structure
 *
 * RL profile function to add, query, or remove profile(s)
 */
enum ice_status
ice_aq_rl_profile(struct ice_hw *hw, enum ice_adminq_opc opcode,
    uint16_t num_profiles, struct ice_aqc_rl_profile_elem *buf,
    uint16_t buf_size, uint16_t *num_processed, struct ice_sq_cd *cd)
{
	struct ice_aqc_rl_profile *cmd;
	struct ice_aq_desc desc;
	enum ice_status status;

	cmd = &desc.params.rl_profile;

	ice_fill_dflt_direct_cmd_desc(&desc, opcode);
	desc.flags |= htole16(ICE_AQ_FLAG_RD);
	cmd->num_profiles = htole16(num_profiles);
	status = ice_aq_send_cmd(hw, &desc, buf, buf_size, cd);
	if (!status && num_processed)
		*num_processed = le16toh(cmd->num_processed);
	return status;
}

/**
 * ice_aq_remove_rl_profile - removes RL profile(s)
 * @hw: pointer to the HW struct
 * @num_profiles: the number of profile(s) to remove
 * @buf: pointer to buffer
 * @buf_size: buffer size in bytes
 * @num_profiles_removed: total number of profiles removed to return
 * @cd: pointer to command details structure or NULL
 *
 * Remove RL profile (0x0415)
 */
enum ice_status
ice_aq_remove_rl_profile(struct ice_hw *hw, uint16_t num_profiles,
			 struct ice_aqc_rl_profile_elem *buf, uint16_t buf_size,
			 uint16_t *num_profiles_removed, struct ice_sq_cd *cd)
{
	return ice_aq_rl_profile(hw, ice_aqc_opc_remove_rl_profiles,
				 num_profiles, buf, buf_size,
				 num_profiles_removed, cd);
}

/**
 * ice_sched_del_rl_profile - remove RL profile
 * @hw: pointer to the HW struct
 * @rl_head: list head
 * @rl_info: rate limit profile information
 *
 * If the profile ID is not referenced anymore, it removes profile ID with
 * its associated parameters from HW DB,and locally. The caller needs to
 * hold scheduler lock.
 */
enum ice_status
ice_sched_del_rl_profile(struct ice_hw *hw,
    struct ice_rl_prof_list_head *list_head,
    struct ice_aqc_rl_profile_info *rl_info)
{
	struct ice_aqc_rl_profile_elem *buf;
	uint16_t num_profiles_removed;
	enum ice_status status;
	uint16_t num_profiles = 1;

	if (rl_info->prof_id_ref != 0)
		return ICE_ERR_IN_USE;

	/* Safe to remove profile ID */
	buf = &rl_info->profile;
	status = ice_aq_remove_rl_profile(hw, num_profiles, buf, sizeof(*buf),
					  &num_profiles_removed, NULL);
	if (status || num_profiles_removed != num_profiles)
		return ICE_ERR_CFG;

	/* Delete stale entry now */
	TAILQ_REMOVE(list_head, rl_info, list_entry);
	ice_free(hw, rl_info);
	return status;
}

/**
 * ice_sched_clear_rl_prof - clears RL prof entries
 * @pi: port information structure
 *
 * This function removes all RL profile from HW as well as from SW DB.
 */
void
ice_sched_clear_rl_prof(struct ice_port_info *pi)
{
	uint16_t ln;
	struct ice_hw *hw = pi->hw;

	for (ln = 0; ln < hw->num_tx_sched_layers; ln++) {
		struct ice_aqc_rl_profile_info *rl_prof_elem;
		struct ice_aqc_rl_profile_info *rl_prof_tmp;

		TAILQ_FOREACH_SAFE(rl_prof_elem, &hw->rl_prof_list[ln],
		    list_entry, rl_prof_tmp) {
			enum ice_status status;

			rl_prof_elem->prof_id_ref = 0;
			status = ice_sched_del_rl_profile(hw,
			    &hw->rl_prof_list[ln], rl_prof_elem);
			if (status) {
				DNPRINTF(ICE_DBG_SCHED,
				    "Remove rl profile failed\n");
				/* On error, free mem required */
				TAILQ_REMOVE(&hw->rl_prof_list[ln],
				    rl_prof_elem, list_entry);
				ice_free(hw, rl_prof_elem);
			}
		}
	}
}

/**
 * ice_sched_clear_tx_topo - clears the scheduler tree nodes
 * @pi: port information structure
 *
 * This function removes all the nodes from HW as well as from SW DB.
 */
void
ice_sched_clear_tx_topo(struct ice_port_info *pi)
{
	if (!pi)
		return;
	/* remove RL profiles related lists */
	ice_sched_clear_rl_prof(pi);
	if (pi->root) {
		ice_free_sched_node(pi, pi->root);
		pi->root = NULL;
	}
}

/**
 * ice_sched_clear_port - clear the scheduler elements from SW DB for a port
 * @pi: port information structure
 *
 * Cleanup scheduling elements from SW DB
 */
void
ice_sched_clear_port(struct ice_port_info *pi)
{
	if (!pi || pi->port_state != ICE_SCHED_PORT_STATE_READY)
		return;

	pi->port_state = ICE_SCHED_PORT_STATE_INIT;
#if 0
	ice_acquire_lock(&pi->sched_lock);
#endif
	ice_sched_clear_tx_topo(pi);
#if 0
	ice_release_lock(&pi->sched_lock);
	ice_destroy_lock(&pi->sched_lock);
#endif
}

/**
 * ice_sched_cleanup_all - cleanup scheduler elements from SW DB for all ports
 * @hw: pointer to the HW struct
 *
 * Cleanup scheduling elements from SW DB for all the ports
 */
void
ice_sched_cleanup_all(struct ice_hw *hw)
{
	if (!hw)
		return;

	if (hw->layer_info) {
		ice_free(hw, hw->layer_info);
		hw->layer_info = NULL;
	}

	ice_sched_clear_port(hw->port_info);

	hw->num_tx_sched_layers = 0;
	hw->num_tx_sched_phys_layers = 0;
	hw->flattened_layers = 0;
	hw->max_cgds = 0;
}

/**
 * ice_is_fw_min_ver
 * @hw: pointer to the hardware structure
 * @branch: branch version
 * @maj: major version
 * @min: minor version
 * @patch: patch version
 *
 * Checks if the firmware is minimum version
 */
bool
ice_is_fw_min_ver(struct ice_hw *hw, uint8_t branch, uint8_t maj, uint8_t min,
    uint8_t patch)
{
	if (hw->fw_branch == branch) {
		if (hw->fw_maj_ver > maj)
			return true;
		if (hw->fw_maj_ver == maj) {
			if (hw->fw_min_ver > min)
				return true;
			if (hw->fw_min_ver == min && hw->fw_patch >= patch)
				return true;
		}
	}

	return false;
}

/**
 * ice_is_fw_api_min_ver
 * @hw: pointer to the hardware structure
 * @maj: major version
 * @min: minor version
 * @patch: patch version
 *
 * Checks if the firmware is minimum version
 */
bool
ice_is_fw_api_min_ver(struct ice_hw *hw, uint8_t maj, uint8_t min,
    uint8_t patch)
{
	if (hw->api_maj_ver == maj) {
		if (hw->api_min_ver > min)
			return true;
		if (hw->api_min_ver == min && hw->api_patch >= patch)
			return true;
	} else if (hw->api_maj_ver > maj) {
		return true;
	}

	return false;
}

/**
 * ice_fw_supports_report_dflt_cfg
 * @hw: pointer to the hardware structure
 *
 * Checks if the firmware supports report default configuration
 */
bool
ice_fw_supports_report_dflt_cfg(struct ice_hw *hw)
{
	return ice_is_fw_api_min_ver(hw, ICE_FW_API_REPORT_DFLT_CFG_MAJ,
				     ICE_FW_API_REPORT_DFLT_CFG_MIN,
				     ICE_FW_API_REPORT_DFLT_CFG_PATCH);
}

/**
 * ice_fw_supports_link_override
 * @hw: pointer to the hardware structure
 *
 * Checks if the firmware supports link override
 */
bool
ice_fw_supports_link_override(struct ice_hw *hw)
{
	return ice_is_fw_api_min_ver(hw, ICE_FW_API_LINK_OVERRIDE_MAJ,
				     ICE_FW_API_LINK_OVERRIDE_MIN,
				     ICE_FW_API_LINK_OVERRIDE_PATCH);
}

#define ICE_PF_RESET_WAIT_COUNT	500

#define ice_arr_elem_idx(idx, val)	[(idx)] = (val)

#ifdef ICE_DEBUG
static const char * const ice_link_mode_str_low[] = {
	ice_arr_elem_idx(0, "100BASE_TX"),
	ice_arr_elem_idx(1, "100M_SGMII"),
	ice_arr_elem_idx(2, "1000BASE_T"),
	ice_arr_elem_idx(3, "1000BASE_SX"),
	ice_arr_elem_idx(4, "1000BASE_LX"),
	ice_arr_elem_idx(5, "1000BASE_KX"),
	ice_arr_elem_idx(6, "1G_SGMII"),
	ice_arr_elem_idx(7, "2500BASE_T"),
	ice_arr_elem_idx(8, "2500BASE_X"),
	ice_arr_elem_idx(9, "2500BASE_KX"),
	ice_arr_elem_idx(10, "5GBASE_T"),
	ice_arr_elem_idx(11, "5GBASE_KR"),
	ice_arr_elem_idx(12, "10GBASE_T"),
	ice_arr_elem_idx(13, "10G_SFI_DA"),
	ice_arr_elem_idx(14, "10GBASE_SR"),
	ice_arr_elem_idx(15, "10GBASE_LR"),
	ice_arr_elem_idx(16, "10GBASE_KR_CR1"),
	ice_arr_elem_idx(17, "10G_SFI_AOC_ACC"),
	ice_arr_elem_idx(18, "10G_SFI_C2C"),
	ice_arr_elem_idx(19, "25GBASE_T"),
	ice_arr_elem_idx(20, "25GBASE_CR"),
	ice_arr_elem_idx(21, "25GBASE_CR_S"),
	ice_arr_elem_idx(22, "25GBASE_CR1"),
	ice_arr_elem_idx(23, "25GBASE_SR"),
	ice_arr_elem_idx(24, "25GBASE_LR"),
	ice_arr_elem_idx(25, "25GBASE_KR"),
	ice_arr_elem_idx(26, "25GBASE_KR_S"),
	ice_arr_elem_idx(27, "25GBASE_KR1"),
	ice_arr_elem_idx(28, "25G_AUI_AOC_ACC"),
	ice_arr_elem_idx(29, "25G_AUI_C2C"),
	ice_arr_elem_idx(30, "40GBASE_CR4"),
	ice_arr_elem_idx(31, "40GBASE_SR4"),
	ice_arr_elem_idx(32, "40GBASE_LR4"),
	ice_arr_elem_idx(33, "40GBASE_KR4"),
	ice_arr_elem_idx(34, "40G_XLAUI_AOC_ACC"),
	ice_arr_elem_idx(35, "40G_XLAUI"),
	ice_arr_elem_idx(36, "50GBASE_CR2"),
	ice_arr_elem_idx(37, "50GBASE_SR2"),
	ice_arr_elem_idx(38, "50GBASE_LR2"),
	ice_arr_elem_idx(39, "50GBASE_KR2"),
	ice_arr_elem_idx(40, "50G_LAUI2_AOC_ACC"),
	ice_arr_elem_idx(41, "50G_LAUI2"),
	ice_arr_elem_idx(42, "50G_AUI2_AOC_ACC"),
	ice_arr_elem_idx(43, "50G_AUI2"),
	ice_arr_elem_idx(44, "50GBASE_CP"),
	ice_arr_elem_idx(45, "50GBASE_SR"),
	ice_arr_elem_idx(46, "50GBASE_FR"),
	ice_arr_elem_idx(47, "50GBASE_LR"),
	ice_arr_elem_idx(48, "50GBASE_KR_PAM4"),
	ice_arr_elem_idx(49, "50G_AUI1_AOC_ACC"),
	ice_arr_elem_idx(50, "50G_AUI1"),
	ice_arr_elem_idx(51, "100GBASE_CR4"),
	ice_arr_elem_idx(52, "100GBASE_SR4"),
	ice_arr_elem_idx(53, "100GBASE_LR4"),
	ice_arr_elem_idx(54, "100GBASE_KR4"),
	ice_arr_elem_idx(55, "100G_CAUI4_AOC_ACC"),
	ice_arr_elem_idx(56, "100G_CAUI4"),
	ice_arr_elem_idx(57, "100G_AUI4_AOC_ACC"),
	ice_arr_elem_idx(58, "100G_AUI4"),
	ice_arr_elem_idx(59, "100GBASE_CR_PAM4"),
	ice_arr_elem_idx(60, "100GBASE_KR_PAM4"),
	ice_arr_elem_idx(61, "100GBASE_CP2"),
	ice_arr_elem_idx(62, "100GBASE_SR2"),
	ice_arr_elem_idx(63, "100GBASE_DR"),
};

static const char * const ice_link_mode_str_high[] = {
	ice_arr_elem_idx(0, "100GBASE_KR2_PAM4"),
	ice_arr_elem_idx(1, "100G_CAUI2_AOC_ACC"),
	ice_arr_elem_idx(2, "100G_CAUI2"),
	ice_arr_elem_idx(3, "100G_AUI2_AOC_ACC"),
	ice_arr_elem_idx(4, "100G_AUI2"),
};
#endif

/**
 * ice_dump_phy_type - helper function to dump phy_type
 * @hw: pointer to the HW structure
 * @low: 64 bit value for phy_type_low
 * @high: 64 bit value for phy_type_high
 * @prefix: prefix string to differentiate multiple dumps
 */
void
ice_dump_phy_type(struct ice_hw *hw, uint64_t low, uint64_t high,
    const char *prefix)
{
#ifdef ICE_DEBUG
	uint32_t i;

	DNPRINTF(ICE_DBG_PHY, "%s: phy_type_low: 0x%016llx\n", prefix,
		  (unsigned long long)low);

	for (i = 0; i < nitems(ice_link_mode_str_low); i++) {
		if (low & (1ULL << i))
			DNPRINTF(ICE_DBG_PHY, "%s:   bit(%d): %s\n",
				  prefix, i, ice_link_mode_str_low[i]);
	}

	DNPRINTF(ICE_DBG_PHY, "%s: phy_type_high: 0x%016llx\n", prefix,
		  (unsigned long long)high);

	for (i = 0; i < nitems(ice_link_mode_str_high); i++) {
		if (high & (1ULL << i))
			DNPRINTF(ICE_DBG_PHY, "%s:   bit(%d): %s\n",
				  prefix, i, ice_link_mode_str_high[i]);
	}
#endif
}

/**
 * ice_phy_maps_to_media
 * @phy_type_low: PHY type low bits
 * @phy_type_high: PHY type high bits
 * @media_mask_low: media type PHY type low bitmask
 * @media_mask_high: media type PHY type high bitmask
 *
 * Return true if PHY type [low|high] bits are only of media type PHY types
 * [low|high] bitmask.
 */
bool
ice_phy_maps_to_media(uint64_t phy_type_low, uint64_t phy_type_high,
		      uint64_t media_mask_low, uint64_t media_mask_high)
{
	/* check if a PHY type exist for media type */
	if (!(phy_type_low & media_mask_low ||
	      phy_type_high & media_mask_high))
		return false;

	/* check that PHY types are only of media type */
	if (!(phy_type_low & ~media_mask_low) &&
	    !(phy_type_high & ~media_mask_high))
		return true;

	return false;
}

/**
 * ice_set_media_type - Sets media type
 * @pi: port information structure
 *
 * Set ice_port_info PHY media type based on PHY type. This should be called
 * from Get PHY caps with media.
 */
void
ice_set_media_type(struct ice_port_info *pi)
{
	enum ice_media_type *media_type;
	uint64_t phy_type_high, phy_type_low;

	phy_type_high = pi->phy.phy_type_high;
	phy_type_low = pi->phy.phy_type_low;
	media_type = &pi->phy.media_type;

	/* if no media, then media type is NONE */
	if (!(pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE))
		*media_type = ICE_MEDIA_NONE;
	/* else if PHY types are only BASE-T, then media type is BASET */
	else if (ice_phy_maps_to_media(phy_type_low, phy_type_high,
				       ICE_MEDIA_BASET_PHY_TYPE_LOW_M, 0))
		*media_type = ICE_MEDIA_BASET;
	/* else if any PHY type is BACKPLANE, then media type is BACKPLANE */
	else if (phy_type_low & ICE_MEDIA_BP_PHY_TYPE_LOW_M ||
		 phy_type_high & ICE_MEDIA_BP_PHY_TYPE_HIGH_M)
		*media_type = ICE_MEDIA_BACKPLANE;
	/* else if PHY types are only optical, or optical and C2M, then media
	 * type is FIBER
	 */
	else if (ice_phy_maps_to_media(phy_type_low, phy_type_high,
				       ICE_MEDIA_OPT_PHY_TYPE_LOW_M, 0) ||
		 (phy_type_low & ICE_MEDIA_OPT_PHY_TYPE_LOW_M &&
		  phy_type_low & ICE_MEDIA_C2M_PHY_TYPE_LOW_M))
		*media_type = ICE_MEDIA_FIBER;
	/* else if PHY types are only DA, or DA and C2C, then media type DA */
	else if (ice_phy_maps_to_media(phy_type_low, phy_type_high,
				       ICE_MEDIA_DAC_PHY_TYPE_LOW_M, 0) ||
		 (phy_type_low & ICE_MEDIA_DAC_PHY_TYPE_LOW_M &&
		 (phy_type_low & ICE_MEDIA_C2C_PHY_TYPE_LOW_M ||
		  phy_type_high & ICE_MEDIA_C2C_PHY_TYPE_HIGH_M)))
		*media_type = ICE_MEDIA_DA;
	/* else if PHY types are only C2M or only C2C, then media is AUI */
	else if (ice_phy_maps_to_media(phy_type_low, phy_type_high,
				       ICE_MEDIA_C2M_PHY_TYPE_LOW_M,
				       ICE_MEDIA_C2M_PHY_TYPE_HIGH_M) ||
		 ice_phy_maps_to_media(phy_type_low, phy_type_high,
				       ICE_MEDIA_C2C_PHY_TYPE_LOW_M,
				       ICE_MEDIA_C2C_PHY_TYPE_HIGH_M))
		*media_type = ICE_MEDIA_AUI;

	else
		*media_type = ICE_MEDIA_UNKNOWN;
}

/**
 * ice_aq_get_phy_caps - returns PHY capabilities
 * @pi: port information structure
 * @qual_mods: report qualified modules
 * @report_mode: report mode capabilities
 * @pcaps: structure for PHY capabilities to be filled
 * @cd: pointer to command details structure or NULL
 *
 * Returns the various PHY capabilities supported on the Port (0x0600)
 */
enum ice_status
ice_aq_get_phy_caps(struct ice_port_info *pi, bool qual_mods,
    uint8_t report_mode, struct ice_aqc_get_phy_caps_data *pcaps,
    struct ice_sq_cd *cd)
{
	struct ice_aqc_get_phy_caps *cmd;
	uint16_t pcaps_size = sizeof(*pcaps);
	struct ice_aq_desc desc;
	enum ice_status status;
	const char *prefix;
	struct ice_hw *hw;

	cmd = &desc.params.get_phy;

	if (!pcaps || (report_mode & ~ICE_AQC_REPORT_MODE_M) || !pi)
		return ICE_ERR_PARAM;
	hw = pi->hw;

	if (report_mode == ICE_AQC_REPORT_DFLT_CFG &&
	    !ice_fw_supports_report_dflt_cfg(hw))
		return ICE_ERR_PARAM;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_get_phy_caps);

	if (qual_mods)
		cmd->param0 |= htole16(ICE_AQC_GET_PHY_RQM);

	cmd->param0 |= htole16(report_mode);

	status = ice_aq_send_cmd(hw, &desc, pcaps, pcaps_size, cd);

	DNPRINTF(ICE_DBG_LINK, "get phy caps dump\n");

	switch (report_mode) {
	case ICE_AQC_REPORT_TOPO_CAP_MEDIA:
		prefix = "phy_caps_media";
		break;
	case ICE_AQC_REPORT_TOPO_CAP_NO_MEDIA:
		prefix = "phy_caps_no_media";
		break;
	case ICE_AQC_REPORT_ACTIVE_CFG:
		prefix = "phy_caps_active";
		break;
	case ICE_AQC_REPORT_DFLT_CFG:
		prefix = "phy_caps_default";
		break;
	default:
		prefix = "phy_caps_invalid";
	}

	ice_dump_phy_type(hw, le64toh(pcaps->phy_type_low),
			  le64toh(pcaps->phy_type_high), prefix);

	DNPRINTF(ICE_DBG_LINK, "%s: report_mode = 0x%x\n",
		  prefix, report_mode);
	DNPRINTF(ICE_DBG_LINK, "%s: caps = 0x%x\n", prefix, pcaps->caps);
	DNPRINTF(ICE_DBG_LINK, "%s: low_power_ctrl_an = 0x%x\n", prefix,
		  pcaps->low_power_ctrl_an);
	DNPRINTF(ICE_DBG_LINK, "%s: eee_cap = 0x%x\n", prefix,
		  pcaps->eee_cap);
	DNPRINTF(ICE_DBG_LINK, "%s: eeer_value = 0x%x\n", prefix,
		  pcaps->eeer_value);
	DNPRINTF(ICE_DBG_LINK, "%s: link_fec_options = 0x%x\n", prefix,
		  pcaps->link_fec_options);
	DNPRINTF(ICE_DBG_LINK, "%s: module_compliance_enforcement = 0x%x\n",
		  prefix, pcaps->module_compliance_enforcement);
	DNPRINTF(ICE_DBG_LINK, "%s: extended_compliance_code = 0x%x\n",
		  prefix, pcaps->extended_compliance_code);
	DNPRINTF(ICE_DBG_LINK, "%s: module_type[0] = 0x%x\n", prefix,
		  pcaps->module_type[0]);
	DNPRINTF(ICE_DBG_LINK, "%s: module_type[1] = 0x%x\n", prefix,
		  pcaps->module_type[1]);
	DNPRINTF(ICE_DBG_LINK, "%s: module_type[2] = 0x%x\n", prefix,
		  pcaps->module_type[2]);

	if (status == ICE_SUCCESS && report_mode == ICE_AQC_REPORT_TOPO_CAP_MEDIA) {
		pi->phy.phy_type_low = le64toh(pcaps->phy_type_low);
		pi->phy.phy_type_high = le64toh(pcaps->phy_type_high);
		memcpy(pi->phy.link_info.module_type, &pcaps->module_type,
		    sizeof(pi->phy.link_info.module_type));
		ice_set_media_type(pi);
		DNPRINTF(ICE_DBG_LINK, "%s: media_type = 0x%x\n", prefix,
			  pi->phy.media_type);
	}

	return status;
}

/**
 * ice_aq_get_link_info
 * @pi: port information structure
 * @ena_lse: enable/disable LinkStatusEvent reporting
 * @link: pointer to link status structure - optional
 * @cd: pointer to command details structure or NULL
 *
 * Get Link Status (0x607). Returns the link status of the adapter.
 */
enum ice_status
ice_aq_get_link_info(struct ice_port_info *pi, bool ena_lse,
		     struct ice_link_status *link, struct ice_sq_cd *cd)
{
	struct ice_aqc_get_link_status_data link_data = { 0 };
	struct ice_aqc_get_link_status *resp;
	struct ice_link_status *li_old, *li;
	struct ice_fc_info *hw_fc_info;
	bool tx_pause, rx_pause;
	struct ice_aq_desc desc;
	enum ice_status status;
	struct ice_hw *hw;
	uint16_t cmd_flags;

	if (!pi)
		return ICE_ERR_PARAM;
	hw = pi->hw;

	li_old = &pi->phy.link_info_old;
	li = &pi->phy.link_info;
	hw_fc_info = &pi->fc;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_get_link_status);
	cmd_flags = (ena_lse) ? ICE_AQ_LSE_ENA : ICE_AQ_LSE_DIS;
	resp = &desc.params.get_link_status;
	resp->cmd_flags = htole16(cmd_flags);
	resp->lport_num = pi->lport;

	status = ice_aq_send_cmd(hw, &desc, &link_data,
	    ICE_GET_LINK_STATUS_DATALEN_V1, cd);
	if (status != ICE_SUCCESS)
		return status;

	/* save off old link status information */
	*li_old = *li;

	/* update current link status information */
	li->link_speed = le16toh(link_data.link_speed);
	li->phy_type_low = le64toh(link_data.phy_type_low);
	li->phy_type_high = le64toh(link_data.phy_type_high);
	li->link_info = link_data.link_info;
	li->link_cfg_err = link_data.link_cfg_err;
	li->an_info = link_data.an_info;
	li->ext_info = link_data.ext_info;
	li->max_frame_size = le16toh(link_data.max_frame_size);
	li->fec_info = link_data.cfg & ICE_AQ_FEC_MASK;
	li->topo_media_conflict = link_data.topo_media_conflict;
	li->pacing = link_data.cfg & (ICE_AQ_CFG_PACING_M |
				      ICE_AQ_CFG_PACING_TYPE_M);

	/* update fc info */
	tx_pause = !!(link_data.an_info & ICE_AQ_LINK_PAUSE_TX);
	rx_pause = !!(link_data.an_info & ICE_AQ_LINK_PAUSE_RX);
	if (tx_pause && rx_pause)
		hw_fc_info->current_mode = ICE_FC_FULL;
	else if (tx_pause)
		hw_fc_info->current_mode = ICE_FC_TX_PAUSE;
	else if (rx_pause)
		hw_fc_info->current_mode = ICE_FC_RX_PAUSE;
	else
		hw_fc_info->current_mode = ICE_FC_NONE;

	li->lse_ena = !!(resp->cmd_flags & htole16(ICE_AQ_LSE_IS_ENABLED));

	DNPRINTF(ICE_DBG_LINK, "get link info\n");
	DNPRINTF(ICE_DBG_LINK, "	link_speed = 0x%x\n", li->link_speed);
	DNPRINTF(ICE_DBG_LINK, "	phy_type_low = 0x%llx\n",
		  (unsigned long long)li->phy_type_low);
	DNPRINTF(ICE_DBG_LINK, "	phy_type_high = 0x%llx\n",
		  (unsigned long long)li->phy_type_high);
	DNPRINTF(ICE_DBG_LINK, "	link_info = 0x%x\n", li->link_info);
	DNPRINTF(ICE_DBG_LINK, "	link_cfg_err = 0x%x\n", li->link_cfg_err);
	DNPRINTF(ICE_DBG_LINK, "	an_info = 0x%x\n", li->an_info);
	DNPRINTF(ICE_DBG_LINK, "	ext_info = 0x%x\n", li->ext_info);
	DNPRINTF(ICE_DBG_LINK, "	fec_info = 0x%x\n", li->fec_info);
	DNPRINTF(ICE_DBG_LINK, "	lse_ena = 0x%x\n", li->lse_ena);
	DNPRINTF(ICE_DBG_LINK, "	max_frame = 0x%x\n",
		  li->max_frame_size);
	DNPRINTF(ICE_DBG_LINK, "	pacing = 0x%x\n", li->pacing);

	/* save link status information */
	if (link)
		*link = *li;

	/* flag cleared so calling functions don't call AQ again */
	pi->phy.get_link_info = false;

	return ICE_SUCCESS;
}

/**
 * ice_cfg_rl_burst_size - Set burst size value
 * @hw: pointer to the HW struct
 * @bytes: burst size in bytes
 *
 * This function configures/set the burst size to requested new value. The new
 * burst size value is used for future rate limit calls. It doesn't change the
 * existing or previously created RL profiles.
 */
enum ice_status
ice_cfg_rl_burst_size(struct ice_hw *hw, uint32_t bytes)
{
	uint16_t burst_size_to_prog;

	if (bytes < ICE_MIN_BURST_SIZE_ALLOWED ||
	    bytes > ICE_MAX_BURST_SIZE_ALLOWED)
		return ICE_ERR_PARAM;
	if (ice_round_to_num(bytes, 64) <=
	    ICE_MAX_BURST_SIZE_64_BYTE_GRANULARITY) {
		/* 64 byte granularity case */
		/* Disable MSB granularity bit */
		burst_size_to_prog = ICE_64_BYTE_GRANULARITY;
		/* round number to nearest 64 byte granularity */
		bytes = ice_round_to_num(bytes, 64);
		/* The value is in 64 byte chunks */
		burst_size_to_prog |= (uint16_t)(bytes / 64);
	} else {
		/* k bytes granularity case */
		/* Enable MSB granularity bit */
		burst_size_to_prog = ICE_KBYTE_GRANULARITY;
		/* round number to nearest 1024 granularity */
		bytes = ice_round_to_num(bytes, 1024);
		/* check rounding doesn't go beyond allowed */
		if (bytes > ICE_MAX_BURST_SIZE_KBYTE_GRANULARITY)
			bytes = ICE_MAX_BURST_SIZE_KBYTE_GRANULARITY;
		/* The value is in k bytes */
		burst_size_to_prog |= (uint16_t)(bytes / 1024);
	}
	hw->max_burst_size = burst_size_to_prog;
	return ICE_SUCCESS;
}

/**
 * ice_init_def_sw_recp - initialize the recipe book keeping tables
 * @hw: pointer to the HW struct
 * @recp_list: pointer to sw recipe list
 *
 * Allocate memory for the entire recipe table and initialize the structures/
 * entries corresponding to basic recipes.
 */
enum ice_status
ice_init_def_sw_recp(struct ice_hw *hw, struct ice_sw_recipe **recp_list)
{
	struct ice_sw_recipe *recps;
	uint8_t i;

	recps = (struct ice_sw_recipe *)
		ice_calloc(hw, ICE_MAX_NUM_RECIPES, sizeof(*recps));
	if (!recps)
		return ICE_ERR_NO_MEMORY;

	for (i = 0; i < ICE_MAX_NUM_RECIPES; i++) {
		recps[i].root_rid = i;
		TAILQ_INIT(&recps[i].filt_rules);
		TAILQ_INIT(&recps[i].adv_filt_rules);
		TAILQ_INIT(&recps[i].filt_replay_rules);
		TAILQ_INIT(&recps[i].rg_list);
		ice_init_lock(&recps[i].filt_rule_lock);
	}

	*recp_list = recps;

	return ICE_SUCCESS;
}

/**
 * ice_init_fltr_mgmt_struct - initializes filter management list and locks
 * @hw: pointer to the HW struct
 */
enum ice_status
ice_init_fltr_mgmt_struct(struct ice_hw *hw)
{
	struct ice_switch_info *sw;
	enum ice_status status;

	hw->switch_info = (struct ice_switch_info *)
			  ice_malloc(hw, sizeof(*hw->switch_info));

	sw = hw->switch_info;

	if (!sw)
		return ICE_ERR_NO_MEMORY;

	TAILQ_INIT(&sw->vsi_list_map_head);
	sw->prof_res_bm_init = 0;

	status = ice_init_def_sw_recp(hw, &hw->switch_info->recp_list);
	if (status) {
		ice_free(hw, hw->switch_info);
		return status;
	}
	return ICE_SUCCESS;
}

/**
 * ice_aq_manage_mac_read - manage MAC address read command
 * @hw: pointer to the HW struct
 * @buf: a virtual buffer to hold the manage MAC read response
 * @buf_size: Size of the virtual buffer
 * @cd: pointer to command details structure or NULL
 *
 * This function is used to return per PF station MAC address (0x0107).
 * NOTE: Upon successful completion of this command, MAC address information
 * is returned in user specified buffer. Please interpret user specified
 * buffer as "manage_mac_read" response.
 * Response such as various MAC addresses are stored in HW struct (port.mac)
 * ice_discover_dev_caps is expected to be called before this function is
 * called.
 */
enum ice_status
ice_aq_manage_mac_read(struct ice_hw *hw, void *buf, uint16_t buf_size,
		       struct ice_sq_cd *cd)
{
	struct ice_aqc_manage_mac_read_resp *resp;
	struct ice_aqc_manage_mac_read *cmd;
	struct ice_aq_desc desc;
	enum ice_status status;
	uint16_t flags;
	uint8_t i;

	cmd = &desc.params.mac_read;

	if (buf_size < sizeof(*resp))
		return ICE_ERR_BUF_TOO_SHORT;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_manage_mac_read);

	status = ice_aq_send_cmd(hw, &desc, buf, buf_size, cd);
	if (status)
		return status;

	resp = (struct ice_aqc_manage_mac_read_resp *)buf;
	flags = le16toh(cmd->flags) & ICE_AQC_MAN_MAC_READ_M;

	if (!(flags & ICE_AQC_MAN_MAC_LAN_ADDR_VALID)) {
		DNPRINTF(ICE_DBG_LAN, "got invalid MAC address\n");
		return ICE_ERR_CFG;
	}

	/* A single port can report up to two (LAN and WoL) addresses */
	for (i = 0; i < cmd->num_addr; i++) {
		if (resp[i].addr_type == ICE_AQC_MAN_MAC_ADDR_TYPE_LAN) {
			memcpy(hw->port_info->mac.lan_addr,
				   resp[i].mac_addr, ETHER_ADDR_LEN);
			memcpy(hw->port_info->mac.perm_addr,
				   resp[i].mac_addr, ETHER_ADDR_LEN);
			break;
		}
	}

	return ICE_SUCCESS;
}

/**
 * ice_rem_sw_rule_info
 * @hw: pointer to the hardware structure
 * @rule_head: pointer to the switch list structure that we want to delete
 */
void
ice_rem_sw_rule_info(struct ice_hw *hw, struct ice_fltr_mgmt_list_head *rule_head)
{
	if (!TAILQ_EMPTY(rule_head)) {
		struct ice_fltr_mgmt_list_entry *entry;
		struct ice_fltr_mgmt_list_entry *tmp;

		TAILQ_FOREACH_SAFE(entry, rule_head, list_entry, tmp) {
			TAILQ_REMOVE(rule_head, entry, list_entry);
			ice_free(hw, entry);
		}
	}
}

/**
 * ice_rm_sw_replay_rule_info - helper function to delete filter replay rules
 * @hw: pointer to the HW struct
 * @sw: pointer to switch info struct for which function removes filters
 *
 * Deletes the filter replay rules for given switch
 */
void
ice_rm_sw_replay_rule_info(struct ice_hw *hw, struct ice_switch_info *sw)
{
	uint8_t i;

	if (!sw)
		return;

	for (i = 0; i < ICE_MAX_NUM_RECIPES; i++) {
		if (!TAILQ_EMPTY(&sw->recp_list[i].filt_replay_rules)) {
			struct ice_fltr_mgmt_list_head *l_head;

			l_head = &sw->recp_list[i].filt_replay_rules;
			if (!sw->recp_list[i].adv_rule)
				ice_rem_sw_rule_info(hw, l_head);
		}
	}
}
/**
 * ice_cleanup_fltr_mgmt_single - clears single filter mngt struct
 * @hw: pointer to the HW struct
 * @sw: pointer to switch info struct for which function clears filters
 */
void
ice_cleanup_fltr_mgmt_single(struct ice_hw *hw, struct ice_switch_info *sw)
{
	struct ice_vsi_list_map_info *v_pos_map;
	struct ice_vsi_list_map_info *v_tmp_map;
	struct ice_sw_recipe *recps;
	uint8_t i;

	if (!sw)
		return;

	TAILQ_FOREACH_SAFE(v_pos_map, &sw->vsi_list_map_head, list_entry,
	    v_tmp_map) {
		TAILQ_REMOVE(&sw->vsi_list_map_head, v_pos_map, list_entry);
		ice_free(hw, v_pos_map);
	}
	recps = sw->recp_list;
	for (i = 0; i < ICE_MAX_NUM_RECIPES; i++) {
		struct ice_recp_grp_entry *rg_entry, *tmprg_entry;

		recps[i].root_rid = i;
		TAILQ_FOREACH_SAFE(rg_entry, &recps[i].rg_list, l_entry,
		    tmprg_entry) {
			TAILQ_REMOVE(&recps[i].rg_list, rg_entry, l_entry);
			ice_free(hw, rg_entry);
		}

		if (recps[i].adv_rule) {
			struct ice_adv_fltr_mgmt_list_entry *tmp_entry;
			struct ice_adv_fltr_mgmt_list_entry *lst_itr;
#if 0
			ice_destroy_lock(&recps[i].filt_rule_lock);
#endif
			TAILQ_FOREACH_SAFE(lst_itr, &recps[i].adv_filt_rules,
			    list_entry, tmp_entry) {
				TAILQ_REMOVE(&recps[i].adv_filt_rules, lst_itr,
				    list_entry);
				ice_free(hw, lst_itr->lkups);
				ice_free(hw, lst_itr);
			}
		} else {
			struct ice_fltr_mgmt_list_entry *lst_itr, *tmp_entry;
#if 0
			ice_destroy_lock(&recps[i].filt_rule_lock);
#endif
			TAILQ_FOREACH_SAFE(lst_itr, &recps[i].filt_rules,
			    list_entry, tmp_entry) {
				TAILQ_REMOVE(&recps[i].filt_rules, lst_itr,
				    list_entry);
				ice_free(hw, lst_itr);
			}
		}
		if (recps[i].root_buf)
			ice_free(hw, recps[i].root_buf);
	}
	ice_rm_sw_replay_rule_info(hw, sw);
	ice_free(hw, sw->recp_list);
	ice_free(hw, sw);
}

/**
 * ice_cleanup_fltr_mgmt_struct - cleanup filter management list and locks
 * @hw: pointer to the HW struct
 */
void
ice_cleanup_fltr_mgmt_struct(struct ice_hw *hw)
{
	ice_cleanup_fltr_mgmt_single(hw, hw->switch_info);
}

/**
 * ice_is_fw_auto_drop_supported
 * @hw: pointer to the hardware structure
 *
 * Checks if the firmware supports auto drop feature
 */
bool
ice_is_fw_auto_drop_supported(struct ice_hw *hw)
{
	if (hw->api_maj_ver >= ICE_FW_API_AUTO_DROP_MAJ &&
	    hw->api_min_ver >= ICE_FW_API_AUTO_DROP_MIN)
		return true;
	return false;
}

/**
 * ice_fill_tx_timer_and_fc_thresh
 * @hw: pointer to the HW struct
 * @cmd: pointer to MAC cfg structure
 *
 * Add Tx timer and FC refresh threshold info to Set MAC Config AQ command
 * descriptor
 */
void
ice_fill_tx_timer_and_fc_thresh(struct ice_hw *hw,
				struct ice_aqc_set_mac_cfg *cmd)
{
	uint16_t fc_thres_val, tx_timer_val;
	uint32_t val;

	/* We read back the transmit timer and fc threshold value of
	 * LFC. Thus, we will use index =
	 * PRTMAC_HSEC_CTL_TX_PAUSE_QUANTA_MAX_INDEX.
	 *
	 * Also, because we are operating on transmit timer and fc
	 * threshold of LFC, we don't turn on any bit in tx_tmr_priority
	 */
#define IDX_OF_LFC PRTMAC_HSEC_CTL_TX_PAUSE_QUANTA_MAX_INDEX

	/* Retrieve the transmit timer */
	val = ICE_READ(hw, PRTMAC_HSEC_CTL_TX_PAUSE_QUANTA(IDX_OF_LFC));
	tx_timer_val = val &
		PRTMAC_HSEC_CTL_TX_PAUSE_QUANTA_HSEC_CTL_TX_PAUSE_QUANTA_M;
	cmd->tx_tmr_value = htole16(tx_timer_val);

	/* Retrieve the fc threshold */
	val = ICE_READ(hw, PRTMAC_HSEC_CTL_TX_PAUSE_REFRESH_TIMER(IDX_OF_LFC));
	fc_thres_val = val & PRTMAC_HSEC_CTL_TX_PAUSE_REFRESH_TIMER_M;

	cmd->fc_refresh_threshold = htole16(fc_thres_val);
}

/**
 * ice_aq_set_mac_cfg
 * @hw: pointer to the HW struct
 * @max_frame_size: Maximum Frame Size to be supported
 * @auto_drop: Tell HW to drop packets if TC queue is blocked
 * @cd: pointer to command details structure or NULL
 *
 * Set MAC configuration (0x0603)
 */
enum ice_status
ice_aq_set_mac_cfg(struct ice_hw *hw, uint16_t max_frame_size, bool auto_drop,
		   struct ice_sq_cd *cd)
{
	struct ice_aqc_set_mac_cfg *cmd;
	struct ice_aq_desc desc;

	cmd = &desc.params.set_mac_cfg;

	if (max_frame_size == 0)
		return ICE_ERR_PARAM;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_set_mac_cfg);

	cmd->max_frame_size = htole16(max_frame_size);

	if (ice_is_fw_auto_drop_supported(hw) && auto_drop)
		cmd->drop_opts |= ICE_AQ_SET_MAC_AUTO_DROP_BLOCKING_PKTS;
	ice_fill_tx_timer_and_fc_thresh(hw, cmd);

	return ice_aq_send_cmd(hw, &desc, NULL, 0, cd);
}

/**
 * ice_vsig_free - free VSI group
 * @hw: pointer to the hardware structure
 * @blk: HW block
 * @vsig: VSIG to remove
 *
 * The function will remove all VSIs associated with the input VSIG and move
 * them to the DEFAULT_VSIG and mark the VSIG available.
 */
enum ice_status
ice_vsig_free(struct ice_hw *hw, enum ice_block blk, uint16_t vsig)
{
	struct ice_vsig_prof *dtmp, *del;
	struct ice_vsig_vsi *vsi_cur;
	uint16_t idx;

	idx = vsig & ICE_VSIG_IDX_M;
	if (idx >= ICE_MAX_VSIGS)
		return ICE_ERR_PARAM;

	if (!hw->blk[blk].xlt2.vsig_tbl[idx].in_use)
		return ICE_ERR_DOES_NOT_EXIST;

	hw->blk[blk].xlt2.vsig_tbl[idx].in_use = false;

	vsi_cur = hw->blk[blk].xlt2.vsig_tbl[idx].first_vsi;
	/* If the VSIG has at least 1 VSI then iterate through the
	 * list and remove the VSIs before deleting the group.
	 */
	if (vsi_cur) {
		/* remove all vsis associated with this VSIG XLT2 entry */
		do {
			struct ice_vsig_vsi *tmp = vsi_cur->next_vsi;

			vsi_cur->vsig = ICE_DEFAULT_VSIG;
			vsi_cur->changed = 1;
			vsi_cur->next_vsi = NULL;
			vsi_cur = tmp;
		} while (vsi_cur);

		/* NULL terminate head of VSI list */
		hw->blk[blk].xlt2.vsig_tbl[idx].first_vsi = NULL;
	}

	/* free characteristic list */
	TAILQ_FOREACH_SAFE(del, &hw->blk[blk].xlt2.vsig_tbl[idx].prop_lst,
	    list, dtmp) {
		TAILQ_REMOVE(&hw->blk[blk].xlt2.vsig_tbl[idx].prop_lst, del,
		    list);
		ice_free(hw, del);
	}

	/* if VSIG characteristic list was cleared for reset
	 * re-initialize the list head
	 */
	TAILQ_INIT(&hw->blk[blk].xlt2.vsig_tbl[idx].prop_lst);

	return ICE_SUCCESS;
}

/**
 * ice_free_vsig_tbl - free complete VSIG table entries
 * @hw: pointer to the hardware structure
 * @blk: the HW block on which to free the VSIG table entries
 */
void
ice_free_vsig_tbl(struct ice_hw *hw, enum ice_block blk)
{
	uint16_t i;

	if (!hw->blk[blk].xlt2.vsig_tbl)
		return;

	for (i = 1; i < ICE_MAX_VSIGS; i++)
		if (hw->blk[blk].xlt2.vsig_tbl[i].in_use)
			ice_vsig_free(hw, blk, i);
}

/**
 * ice_free_prof_map - free profile map
 * @hw: pointer to the hardware structure
 * @blk_idx: HW block index
 */
void
ice_free_prof_map(struct ice_hw *hw, uint8_t blk_idx)
{
	struct ice_es *es = &hw->blk[blk_idx].es;
	struct ice_prof_map *del, *tmp;
#if 0
	ice_acquire_lock(&es->prof_map_lock);
#endif
	TAILQ_FOREACH_SAFE(del, &es->prof_map, list, tmp) { 
		TAILQ_REMOVE(&es->prof_map, del, list);
		ice_free(hw, del);
	}
	TAILQ_INIT(&es->prof_map);
#if 0
	ice_release_lock(&es->prof_map_lock);
#endif
}

/**
 * ice_free_flow_profs - free flow profile entries
 * @hw: pointer to the hardware structure
 * @blk_idx: HW block index
 */
void
ice_free_flow_profs(struct ice_hw *hw, uint8_t blk_idx)
{
	struct ice_flow_prof *p, *tmp;
#if 0
	ice_acquire_lock(&hw->fl_profs_locks[blk_idx]);
#endif
	TAILQ_FOREACH_SAFE(p, &hw->fl_profs[blk_idx], l_entry, tmp) {
		TAILQ_REMOVE(&hw->fl_profs[blk_idx], p, l_entry);

		ice_free(hw, p);
	}
#if 0
	ice_release_lock(&hw->fl_profs_locks[blk_idx]);
#endif
	/* if driver is in reset and tables are being cleared
	 * re-initialize the flow profile list heads
	 */
	TAILQ_INIT(&hw->fl_profs[blk_idx]);
}

/**
 * ice_free_hw_tbls - free hardware table memory
 * @hw: pointer to the hardware structure
 */
void
ice_free_hw_tbls(struct ice_hw *hw)
{
	struct ice_rss_cfg *r, *rt;
	uint8_t i;

	for (i = 0; i < ICE_BLK_COUNT; i++) {
		if (hw->blk[i].is_list_init) {
#if 0
			struct ice_es *es = &hw->blk[i].es;
#endif
			ice_free_prof_map(hw, i);
#if 0
			ice_destroy_lock(&es->prof_map_lock);
#endif

			ice_free_flow_profs(hw, i);
#if 0
			ice_destroy_lock(&hw->fl_profs_locks[i]);
#endif

			hw->blk[i].is_list_init = false;
		}
		ice_free_vsig_tbl(hw, (enum ice_block)i);
		ice_free(hw, hw->blk[i].xlt1.ptypes);
		ice_free(hw, hw->blk[i].xlt1.ptg_tbl);
		ice_free(hw, hw->blk[i].xlt1.t);
		ice_free(hw, hw->blk[i].xlt2.t);
		ice_free(hw, hw->blk[i].xlt2.vsig_tbl);
		ice_free(hw, hw->blk[i].xlt2.vsis);
		ice_free(hw, hw->blk[i].prof.t);
		ice_free(hw, hw->blk[i].prof_redir.t);
		ice_free(hw, hw->blk[i].es.t);
		ice_free(hw, hw->blk[i].es.ref_count);
		ice_free(hw, hw->blk[i].es.written);
	}

	TAILQ_FOREACH_SAFE(r, &hw->rss_list_head, l_entry, rt) {
		TAILQ_REMOVE(&hw->rss_list_head, r, l_entry);
		ice_free(hw, r);
	}
#if 0
	ice_destroy_lock(&hw->rss_locks);
#endif
	memset(hw->blk, 0, sizeof(hw->blk));
}

/**
 * ice_init_flow_profs - init flow profile locks and list heads
 * @hw: pointer to the hardware structure
 * @blk_idx: HW block index
 */
void ice_init_flow_profs(struct ice_hw *hw, uint8_t blk_idx)
{
#if 0
	ice_init_lock(&hw->fl_profs_locks[blk_idx]);
#endif
	TAILQ_INIT(&hw->fl_profs[blk_idx]);
}

/* Block / table size info */
struct ice_blk_size_details {
	uint16_t xlt1;			/* # XLT1 entries */
	uint16_t xlt2;			/* # XLT2 entries */
	uint16_t prof_tcam;			/* # profile ID TCAM entries */
	uint16_t prof_id;			/* # profile IDs */
	uint8_t prof_cdid_bits;		/* # CDID one-hot bits used in key */
	uint16_t prof_redir;			/* # profile redirection entries */
	uint16_t es;				/* # extraction sequence entries */
	uint16_t fvw;			/* # field vector words */
	uint8_t overwrite;			/* overwrite existing entries allowed */
	uint8_t reverse;			/* reverse FV order */
};

static const struct ice_blk_size_details blk_sizes[ICE_BLK_COUNT] = {
	/**
	 * Table Definitions
	 * XLT1 - Number of entries in XLT1 table
	 * XLT2 - Number of entries in XLT2 table
	 * TCAM - Number of entries Profile ID TCAM table
	 * CDID - Control Domain ID of the hardware block
	 * PRED - Number of entries in the Profile Redirection Table
	 * FV   - Number of entries in the Field Vector
	 * FVW  - Width (in WORDs) of the Field Vector
	 * OVR  - Overwrite existing table entries
	 * REV  - Reverse FV
	 */
	/*          XLT1        , XLT2        ,TCAM, PID,CDID,PRED,   FV, FVW */
	/*          Overwrite   , Reverse FV */
	/* SW  */ { ICE_XLT1_CNT, ICE_XLT2_CNT, 512, 256,   0,  256, 256,  48,
		    false, false },
	/* ACL */ { ICE_XLT1_CNT, ICE_XLT2_CNT, 512, 128,   0,  128, 128,  32,
		    false, false },
	/* FD  */ { ICE_XLT1_CNT, ICE_XLT2_CNT, 512, 128,   0,  128, 128,  24,
		    false, true  },
	/* RSS */ { ICE_XLT1_CNT, ICE_XLT2_CNT, 512, 128,   0,  128, 128,  24,
		    true,  true  },
	/* PE  */ { ICE_XLT1_CNT, ICE_XLT2_CNT,  64,  32,   0,   32,  32,  24,
		    false, false },
};

enum ice_sid_all {
	ICE_SID_XLT1_OFF = 0,
	ICE_SID_XLT2_OFF,
	ICE_SID_PR_OFF,
	ICE_SID_PR_REDIR_OFF,
	ICE_SID_ES_OFF,
	ICE_SID_OFF_COUNT,
};

/* Block / table section IDs */
static const uint32_t ice_blk_sids[ICE_BLK_COUNT][ICE_SID_OFF_COUNT] = {
	/* SWITCH */
	{	ICE_SID_XLT1_SW,
		ICE_SID_XLT2_SW,
		ICE_SID_PROFID_TCAM_SW,
		ICE_SID_PROFID_REDIR_SW,
		ICE_SID_FLD_VEC_SW
	},

	/* ACL */
	{	ICE_SID_XLT1_ACL,
		ICE_SID_XLT2_ACL,
		ICE_SID_PROFID_TCAM_ACL,
		ICE_SID_PROFID_REDIR_ACL,
		ICE_SID_FLD_VEC_ACL
	},

	/* FD */
	{	ICE_SID_XLT1_FD,
		ICE_SID_XLT2_FD,
		ICE_SID_PROFID_TCAM_FD,
		ICE_SID_PROFID_REDIR_FD,
		ICE_SID_FLD_VEC_FD
	},

	/* RSS */
	{	ICE_SID_XLT1_RSS,
		ICE_SID_XLT2_RSS,
		ICE_SID_PROFID_TCAM_RSS,
		ICE_SID_PROFID_REDIR_RSS,
		ICE_SID_FLD_VEC_RSS
	},

	/* PE */
	{	ICE_SID_XLT1_PE,
		ICE_SID_XLT2_PE,
		ICE_SID_PROFID_TCAM_PE,
		ICE_SID_PROFID_REDIR_PE,
		ICE_SID_FLD_VEC_PE
	}
};

/**
 * ice_init_hw_tbls - init hardware table memory
 * @hw: pointer to the hardware structure
 */
enum ice_status
ice_init_hw_tbls(struct ice_hw *hw)
{
	uint8_t i;
#if 0
	ice_init_lock(&hw->rss_locks);
#endif
	TAILQ_INIT(&hw->rss_list_head);
	for (i = 0; i < ICE_BLK_COUNT; i++) {
		struct ice_prof_redir *prof_redir = &hw->blk[i].prof_redir;
		struct ice_prof_tcam *prof = &hw->blk[i].prof;
		struct ice_xlt1 *xlt1 = &hw->blk[i].xlt1;
		struct ice_xlt2 *xlt2 = &hw->blk[i].xlt2;
		struct ice_es *es = &hw->blk[i].es;
		uint16_t j;

		if (hw->blk[i].is_list_init)
			continue;

		ice_init_flow_profs(hw, i);
		ice_init_lock(&es->prof_map_lock);
		TAILQ_INIT(&es->prof_map);
		hw->blk[i].is_list_init = true;

		hw->blk[i].overwrite = blk_sizes[i].overwrite;
		es->reverse = blk_sizes[i].reverse;

		xlt1->sid = ice_blk_sids[i][ICE_SID_XLT1_OFF];
		xlt1->count = blk_sizes[i].xlt1;

		xlt1->ptypes = (struct ice_ptg_ptype *)
			ice_calloc(hw, xlt1->count, sizeof(*xlt1->ptypes));

		if (!xlt1->ptypes)
			goto err;

		xlt1->ptg_tbl = (struct ice_ptg_entry *)
			ice_calloc(hw, ICE_MAX_PTGS, sizeof(*xlt1->ptg_tbl));

		if (!xlt1->ptg_tbl)
			goto err;

		xlt1->t = (uint8_t *)ice_calloc(hw, xlt1->count,
		    sizeof(*xlt1->t));
		if (!xlt1->t)
			goto err;

		xlt2->sid = ice_blk_sids[i][ICE_SID_XLT2_OFF];
		xlt2->count = blk_sizes[i].xlt2;

		xlt2->vsis = (struct ice_vsig_vsi *)
			ice_calloc(hw, xlt2->count, sizeof(*xlt2->vsis));

		if (!xlt2->vsis)
			goto err;

		xlt2->vsig_tbl = (struct ice_vsig_entry *)
			ice_calloc(hw, xlt2->count, sizeof(*xlt2->vsig_tbl));
		if (!xlt2->vsig_tbl)
			goto err;

		for (j = 0; j < xlt2->count; j++)
			TAILQ_INIT(&xlt2->vsig_tbl[j].prop_lst);

		xlt2->t = (uint16_t *)ice_calloc(hw, xlt2->count,
		    sizeof(*xlt2->t));
		if (!xlt2->t)
			goto err;

		prof->sid = ice_blk_sids[i][ICE_SID_PR_OFF];
		prof->count = blk_sizes[i].prof_tcam;
		prof->max_prof_id = blk_sizes[i].prof_id;
		prof->cdid_bits = blk_sizes[i].prof_cdid_bits;
		prof->t = (struct ice_prof_tcam_entry *)
			ice_calloc(hw, prof->count, sizeof(*prof->t));

		if (!prof->t)
			goto err;

		prof_redir->sid = ice_blk_sids[i][ICE_SID_PR_REDIR_OFF];
		prof_redir->count = blk_sizes[i].prof_redir;
		prof_redir->t = (uint8_t *)ice_calloc(hw, prof_redir->count,
						 sizeof(*prof_redir->t));

		if (!prof_redir->t)
			goto err;

		es->sid = ice_blk_sids[i][ICE_SID_ES_OFF];
		es->count = blk_sizes[i].es;
		es->fvw = blk_sizes[i].fvw;
		es->t = (struct ice_fv_word *)
			ice_calloc(hw, (uint32_t)(es->count * es->fvw),
				   sizeof(*es->t));
		if (!es->t)
			goto err;

		es->ref_count = (uint16_t *)
			ice_calloc(hw, es->count, sizeof(*es->ref_count));

		if (!es->ref_count)
			goto err;

		es->written = (uint8_t *)
			ice_calloc(hw, es->count, sizeof(*es->written));

		if (!es->written)
			goto err;

	}
	return ICE_SUCCESS;

err:
	ice_free_hw_tbls(hw);
	return ICE_ERR_NO_MEMORY;
}

enum ice_status
ice_init_hw(struct ice_hw *hw)
{
	struct ice_softc *sc = hw->hw_sc;
	struct ice_aqc_get_phy_caps_data *pcaps;
	enum ice_status status;
	uint16_t mac_buf_len;
	void *mac_buf;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);

	/* Set MAC type based on DeviceID */
	ice_set_mac_type(hw);

	hw->pf_id = (uint8_t)(ICE_READ(hw, PF_FUNC_RID) &
			 PF_FUNC_RID_FUNCTION_NUMBER_M) >>
		PF_FUNC_RID_FUNCTION_NUMBER_S;

	status = ice_reset(hw, ICE_RESET_PFR);
	if (status)
		return status;

	ice_get_itr_intrl_gran(hw);

	status = ice_create_all_ctrlq(hw);
	if (status)
		goto err_unroll_cqinit;

	ice_fwlog_set_support_ena(hw);
	status = ice_fwlog_set(hw, &hw->fwlog_cfg);
	if (status) {
		DNPRINTF(ICE_DBG_INIT,
		    "Failed to enable FW logging, status %d.\n", status);
	} else {
		if (hw->fwlog_cfg.options & ICE_FWLOG_OPTION_REGISTER_ON_INIT) {
			status = ice_fwlog_register(hw);
			if (status)
				DNPRINTF(ICE_DBG_INIT,
				    "Failed to register for FW logging "
				    "events, status %d.\n", status);
		} else {
			status = ice_fwlog_unregister(hw);
			if (status)
				DNPRINTF(ICE_DBG_INIT, "Failed to unregister "
				    "for FW logging events, status %d.\n",
				    status);
		}
	}

	status = ice_init_nvm(hw);
	if (status)
		goto err_unroll_cqinit;

	if (ice_get_fw_mode(hw) == ICE_FW_MODE_ROLLBACK)
		ice_print_rollback_msg(hw);

	status = ice_clear_pf_cfg(hw);
	if (status)
		goto err_unroll_cqinit;

	ice_clear_pxe_mode(hw);

	status = ice_get_caps(hw);
	if (status)
		goto err_unroll_cqinit;

	if (!hw->port_info)
		hw->port_info = (struct ice_port_info *)
			ice_malloc(hw, sizeof(*hw->port_info));
	if (!hw->port_info) {
		status = ICE_ERR_NO_MEMORY;
		goto err_unroll_cqinit;
	}

	/* set the back pointer to HW */
	hw->port_info->hw = hw;

	/* Initialize port_info struct with switch configuration data */
	status = ice_get_initial_sw_cfg(hw);
	if (status)
		goto err_unroll_alloc;

	hw->evb_veb = true;
	/* Query the allocated resources for Tx scheduler */
	status = ice_sched_query_res_alloc(hw);
	if (status) {
		DNPRINTF(ICE_DBG_SCHED,
		    "Failed to get scheduler allocated resources\n");
		goto err_unroll_alloc;
	}
	ice_sched_get_psm_clk_freq(hw);

	/* Initialize port_info struct with scheduler data */
	status = ice_sched_init_port(hw->port_info);
	if (status)
		goto err_unroll_sched;
	pcaps = (struct ice_aqc_get_phy_caps_data *)
		ice_malloc(hw, sizeof(*pcaps));
	if (!pcaps) {
		status = ICE_ERR_NO_MEMORY;
		goto err_unroll_sched;
	}
	/* Initialize port_info struct with PHY capabilities */
	status = ice_aq_get_phy_caps(hw->port_info, false,
				     ICE_AQC_REPORT_TOPO_CAP_MEDIA, pcaps, NULL);
	ice_free(hw, pcaps);
	if (status)
		printf("%s: Get PHY capabilities failed status = %d, "
		    "continuing anyway\n", sc->sc_dev.dv_xname, status);

	/* Initialize port_info struct with link information */
	status = ice_aq_get_link_info(hw->port_info, false, NULL, NULL);
	if (status)
		goto err_unroll_sched;
	/* need a valid SW entry point to build a Tx tree */
	if (!hw->sw_entry_point_layer) {
		DNPRINTF(ICE_DBG_SCHED, "invalid sw entry point\n");
		status = ICE_ERR_CFG;
		goto err_unroll_sched;
	}

	TAILQ_INIT(&hw->agg_list);
	/* Initialize max burst size */
	if (!hw->max_burst_size)
		ice_cfg_rl_burst_size(hw, ICE_SCHED_DFLT_BURST_SIZE);

	status = ice_init_fltr_mgmt_struct(hw);
	if (status)
		goto err_unroll_sched;

	/* Get MAC information */

	/* A single port can report up to two (LAN and WoL) addresses */
	mac_buf = ice_calloc(hw, 2,
			     sizeof(struct ice_aqc_manage_mac_read_resp));
	mac_buf_len = 2 * sizeof(struct ice_aqc_manage_mac_read_resp);

	if (!mac_buf) {
		status = ICE_ERR_NO_MEMORY;
		goto err_unroll_fltr_mgmt_struct;
	}

	status = ice_aq_manage_mac_read(hw, mac_buf, mac_buf_len, NULL);
	ice_free(hw, mac_buf);

	if (status)
		goto err_unroll_fltr_mgmt_struct;

	/* enable jumbo frame support at MAC level */
	status = ice_aq_set_mac_cfg(hw, ICE_AQ_SET_MAC_FRAME_SIZE_MAX, false,
				    NULL);
	if (status)
		goto err_unroll_fltr_mgmt_struct;

	status = ice_init_hw_tbls(hw);
	if (status)
		goto err_unroll_fltr_mgmt_struct;
#if 0
	ice_init_lock(&hw->tnl_lock);
#endif
	return ICE_SUCCESS;
err_unroll_fltr_mgmt_struct:
	ice_cleanup_fltr_mgmt_struct(hw);
err_unroll_sched:
	ice_sched_cleanup_all(hw);
err_unroll_alloc:
	ice_free(hw, hw->port_info);
	hw->port_info = NULL;
err_unroll_cqinit:
	ice_destroy_all_ctrlq(hw);
	return status;
}

/**
 * ice_deinit_hw - unroll initialization operations done by ice_init_hw
 * @hw: pointer to the hardware structure
 *
 * This should be called only during nominal operation, not as a result of
 * ice_init_hw() failing since ice_init_hw() will take care of unrolling
 * applicable initializations if it fails for any reason.
 */
void ice_deinit_hw(struct ice_hw *hw)
{
	ice_cleanup_fltr_mgmt_struct(hw);

	ice_sched_cleanup_all(hw);
#if 0
	ice_sched_clear_agg(hw);
	ice_free_seg(hw);
#endif
	ice_free_hw_tbls(hw);
#if 0
	ice_destroy_lock(&hw->tnl_lock);
#endif
	if (hw->port_info) {
		ice_free(hw, hw->port_info);
		hw->port_info = NULL;
	}

	ice_destroy_all_ctrlq(hw);
#if 0
	/* Clear VSI contexts if not already cleared */
	ice_clear_all_vsi_ctx(hw);
#endif
}

void
ice_rxfill(struct ice_softc *sc, struct ice_rx_queue *rxq)
{
	union ice_32b_rx_flex_desc *ring, *rxd;
	struct ice_rx_map *rxm;
	bus_dmamap_t map;
	struct mbuf *m;
	unsigned int prod;
	unsigned int slots;
	unsigned int mask;
	int post = 0;

	slots = if_rxr_get(&rxq->rxq_acct, rxq->desc_count);
	if (slots == 0)
		return;

	prod = rxq->rxq_prod;

	ring = ICE_DMA_KVA(&rxq->rx_desc_mem);
	mask = rxq->desc_count - 1;

	do {
		rxm = &rxq->rx_map[prod];

		m = MCLGETL(NULL, M_DONTWAIT, MCLBYTES + ETHER_ALIGN);
		if (m == NULL)
			break;
		m->m_data += (m->m_ext.ext_size - (MCLBYTES + ETHER_ALIGN));
		m->m_len = m->m_pkthdr.len = MCLBYTES + ETHER_ALIGN;

		map = rxm->rxm_map;

		if (bus_dmamap_load_mbuf(sc->sc_dmat, map, m,
		    BUS_DMA_NOWAIT) != 0) {
			m_freem(m);
			break;
		}

		rxm->rxm_m = m;

		bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
		    BUS_DMASYNC_PREREAD);

		rxd = &ring[prod];

		htolem64(&rxd->read.pkt_addr, map->dm_segs[0].ds_addr);
		rxd->read.hdr_addr = htole64(0);

		prod++;
		prod &= mask;

		post = 1;
	} while (--slots);

	if_rxr_put(&rxq->rxq_acct, slots);

	if (if_rxr_inuse(&rxq->rxq_acct) == 0)
		timeout_add(&rxq->rxq_refill, 1);
	else if (post) {
		rxq->rxq_prod = prod;
		ICE_WRITE(&sc->hw, rxq->tail, prod);
	}
}

/**
 * ice_aq_manage_mac_write - manage MAC address write command
 * @hw: pointer to the HW struct
 * @mac_addr: MAC address to be written as LAA/LAA+WoL/Port address
 * @flags: flags to control write behavior
 * @cd: pointer to command details structure or NULL
 *
 * This function is used to write MAC address to the NVM (0x0108).
 */
enum ice_status
ice_aq_manage_mac_write(struct ice_hw *hw, const uint8_t *mac_addr,
    uint8_t flags, struct ice_sq_cd *cd)
{
	struct ice_aqc_manage_mac_write *cmd;
	struct ice_aq_desc desc;

	cmd = &desc.params.mac_write;
	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_manage_mac_write);

	cmd->flags = flags;
	memcpy(cmd->mac_addr, mac_addr, ETHER_ADDR_LEN);

	return ice_aq_send_cmd(hw, &desc, NULL, 0, cd);
}

/**
 * ice_update_laa_mac - Update MAC address if Locally Administered
 * @sc: the device softc
 *
 * Update the device MAC address when a Locally Administered Address is
 * assigned.
 *
 * This function does *not* update the MAC filter list itself. Instead, it
 * should be called after ice_rm_pf_default_mac_filters, so that the previous
 * address filter will be removed, and before ice_cfg_pf_default_mac_filters,
 * so that the new address filter will be assigned.
 */
void
ice_update_laa_mac(struct ice_softc *sc)
{
	struct ifnet *ifp = &sc->sc_ac.ac_if;
	uint8_t *lladdr = ((struct arpcom *)ifp)->ac_enaddr;
	struct ice_hw *hw = &sc->hw;
	enum ice_status status;

	/* Desired address already set in hardware? */
	if (!memcmp(lladdr, hw->port_info->mac.lan_addr, ETHER_ADDR_LEN))
		return;

	status = ice_aq_manage_mac_write(hw, lladdr,
	    ICE_AQC_MAN_MAC_UPDATE_LAA_WOL, NULL);
	if (status) {
		printf("%s: Failed to write mac %s to firmware, "
		    "err %s aq_err %s\n", sc->sc_dev.dv_xname,
		    ether_sprintf(lladdr), ice_status_str(status),
		    ice_aq_str(hw->adminq.sq_last_status));
		return;
	}

	/* Cache current hardware address. */
	memcpy(hw->port_info->mac.lan_addr, lladdr, ETHER_ADDR_LEN);
}

/**
 * ice_add_mac_to_list - Add MAC filter to a MAC filter list
 * @vsi: the VSI to forward to
 * @list: list which contains MAC filter entries
 * @addr: the MAC address to be added
 * @action: filter action to perform on match
 *
 * Adds a MAC address filter to the list which will be forwarded to firmware
 * to add a series of MAC address filters.
 *
 * Returns 0 on success, and an error code on failure.
 *
 */
int
ice_add_mac_to_list(struct ice_vsi *vsi, struct ice_fltr_list_head *list,
		    const uint8_t *addr, enum ice_sw_fwd_act_type action)
{
	struct ice_fltr_list_entry *entry;

	entry = malloc(sizeof(*entry), M_DEVBUF, M_NOWAIT|M_ZERO);
	if (!entry)
		return (ENOMEM);

	entry->fltr_info.flag = ICE_FLTR_TX;
	entry->fltr_info.src_id = ICE_SRC_ID_VSI;
	entry->fltr_info.lkup_type = ICE_SW_LKUP_MAC;
	entry->fltr_info.fltr_act = action;
	entry->fltr_info.vsi_handle = vsi->idx;
	memcpy(entry->fltr_info.l_data.mac.mac_addr, addr, ETHER_ADDR_LEN);

	TAILQ_INSERT_HEAD(list, entry, list_entry);

	return 0;
}

/**
 * ice_free_fltr_list - Free memory associated with a MAC address list
 * @list: the list to free
 *
 * Free the memory of each entry associated with the list.
 */
void
ice_free_fltr_list(struct ice_fltr_list_head *list)
{
	struct ice_fltr_list_entry *e, *tmp;

	TAILQ_FOREACH_SAFE(e, list, list_entry, tmp) {
		TAILQ_REMOVE(list, e, list_entry);
		free(e, M_DEVBUF, sizeof(*e));
	}
}

/**
 * ice_find_rule_entry - Search a rule entry
 * @list_head: head of rule list
 * @f_info: rule information
 *
 * Helper function to search for a given rule entry
 * Returns pointer to entry storing the rule if found
 */
struct ice_fltr_mgmt_list_entry *
ice_find_rule_entry(struct ice_fltr_mgmt_list_head *list_head,
		    struct ice_fltr_info *f_info)
{
	struct ice_fltr_mgmt_list_entry *list_itr, *ret = NULL;

	TAILQ_FOREACH(list_itr, list_head, list_entry) {
		if (!memcmp(&f_info->l_data, &list_itr->fltr_info.l_data,
			    sizeof(f_info->l_data)) &&
		    f_info->flag == list_itr->fltr_info.flag) {
			ret = list_itr;
			break;
		}
	}

	return ret;
}

/* Dummy ethernet header needed in the ice_sw_rule_*
 * struct to configure any switch filter rules.
 * {DA (6 bytes), SA(6 bytes),
 * Ether type (2 bytes for header without VLAN tag) OR
 * VLAN tag (4 bytes for header with VLAN tag) }
 *
 * Word on Hardcoded values
 * byte 0 = 0x2: to identify it as locally administered DA MAC
 * byte 6 = 0x2: to identify it as locally administered SA MAC
 * byte 12 = 0x81 & byte 13 = 0x00:
 *	In case of VLAN filter first two bytes defines ether type (0x8100)
 *	and remaining two bytes are placeholder for programming a given VLAN ID
 *	In case of Ether type filter it is treated as header without VLAN tag
 *	and byte 12 and 13 is used to program a given Ether type instead
 */
static const uint8_t dummy_eth_header[ICE_DUMMY_ETH_HDR_LEN] = {
    0x2, 0, 0, 0, 0, 0, 0x2, 0, 0, 0, 0, 0, 0x81, 0, 0, 0
};

#define ICE_ETH_DA_OFFSET		0
#define ICE_ETH_ETHTYPE_OFFSET		12
#define ICE_ETH_VLAN_TCI_OFFSET		14
#define ICE_MAX_VLAN_ID			0xFFF
#define ICE_IPV6_ETHER_ID		0x86DD
#define ICE_PPP_IPV6_PROTO_ID		0x0057
#define ICE_ETH_P_8021Q			0x8100

/**
 * ice_fill_sw_info - Helper function to populate lb_en and lan_en
 * @hw: pointer to the hardware structure
 * @fi: filter info structure to fill/update
 *
 * This helper function populates the lb_en and lan_en elements of the provided
 * ice_fltr_info struct using the switch's type and characteristics of the
 * switch rule being configured.
 */
void
ice_fill_sw_info(struct ice_hw *hw, struct ice_fltr_info *fi)
{
	fi->lb_en = false;
	fi->lan_en = false;
	if ((fi->flag & ICE_FLTR_TX) &&
	    (fi->fltr_act == ICE_FWD_TO_VSI ||
	     fi->fltr_act == ICE_FWD_TO_VSI_LIST ||
	     fi->fltr_act == ICE_FWD_TO_Q ||
	     fi->fltr_act == ICE_FWD_TO_QGRP)) {
		/* Setting LB for prune actions will result in replicated
		 * packets to the internal switch that will be dropped.
		 */
		if (fi->lkup_type != ICE_SW_LKUP_VLAN)
			fi->lb_en = true;

		/* Set lan_en to TRUE if
		 * 1. The switch is a VEB AND
		 * 2
		 * 2.1 The lookup is a directional lookup like ethertype,
		 * promiscuous, ethertype-MAC, promiscuous-VLAN
		 * and default-port OR
		 * 2.2 The lookup is VLAN, OR
		 * 2.3 The lookup is MAC with mcast or bcast addr for MAC, OR
		 * 2.4 The lookup is MAC_VLAN with mcast or bcast addr for MAC.
		 *
		 * OR
		 *
		 * The switch is a VEPA.
		 *
		 * In all other cases, the LAN enable has to be set to false.
		 */

		if (hw->evb_veb) {
			if (fi->lkup_type == ICE_SW_LKUP_ETHERTYPE ||
			    fi->lkup_type == ICE_SW_LKUP_PROMISC ||
			    fi->lkup_type == ICE_SW_LKUP_ETHERTYPE_MAC ||
			    fi->lkup_type == ICE_SW_LKUP_PROMISC_VLAN ||
			    fi->lkup_type == ICE_SW_LKUP_DFLT ||
			    fi->lkup_type == ICE_SW_LKUP_VLAN ||
			    (fi->lkup_type == ICE_SW_LKUP_MAC &&
			     ETHER_IS_MULTICAST(fi->l_data.mac.mac_addr)) ||
			    (fi->lkup_type == ICE_SW_LKUP_MAC_VLAN &&
			     ETHER_IS_MULTICAST(fi->l_data.mac.mac_addr)))
				fi->lan_en = true;
		} else {
			fi->lan_en = true;
		}
	}

	/* To be able to receive packets coming from the VF on the same PF,
	 * unicast filter needs to be added without LB_EN bit
	 */
	if (fi->flag & ICE_FLTR_RX_LB) {
		fi->lb_en = false;
		fi->lan_en = true;
	}
}

/**
 * ice_fill_sw_rule - Helper function to fill switch rule structure
 * @hw: pointer to the hardware structure
 * @f_info: entry containing packet forwarding information
 * @s_rule: switch rule structure to be filled in based on mac_entry
 * @opc: switch rules population command type - pass in the command opcode
 */
void
ice_fill_sw_rule(struct ice_hw *hw, struct ice_fltr_info *f_info,
		 struct ice_sw_rule_lkup_rx_tx *s_rule,
		 enum ice_adminq_opc opc)
{
	uint16_t vlan_id = ICE_MAX_VLAN_ID + 1;
	uint16_t vlan_tpid = ICE_ETH_P_8021Q;
	void *daddr = NULL;
	uint16_t eth_hdr_sz;
	uint8_t *eth_hdr;
	uint32_t act = 0;
	uint16_t *off;
	uint8_t q_rgn;

	if (opc == ice_aqc_opc_remove_sw_rules) {
		s_rule->act = 0;
		s_rule->index = htole16(f_info->fltr_rule_id);
		s_rule->hdr_len = 0;
		return;
	}

	eth_hdr_sz = sizeof(dummy_eth_header);
	eth_hdr = s_rule->hdr_data;

	/* initialize the ether header with a dummy header */
	memcpy(eth_hdr, dummy_eth_header, eth_hdr_sz);
	ice_fill_sw_info(hw, f_info);

	switch (f_info->fltr_act) {
	case ICE_FWD_TO_VSI:
		act |= (f_info->fwd_id.hw_vsi_id << ICE_SINGLE_ACT_VSI_ID_S) &
			ICE_SINGLE_ACT_VSI_ID_M;
		if (f_info->lkup_type != ICE_SW_LKUP_VLAN)
			act |= ICE_SINGLE_ACT_VSI_FORWARDING |
				ICE_SINGLE_ACT_VALID_BIT;
		break;
	case ICE_FWD_TO_VSI_LIST:
		act |= ICE_SINGLE_ACT_VSI_LIST;
		act |= (f_info->fwd_id.vsi_list_id <<
			ICE_SINGLE_ACT_VSI_LIST_ID_S) &
			ICE_SINGLE_ACT_VSI_LIST_ID_M;
		if (f_info->lkup_type != ICE_SW_LKUP_VLAN)
			act |= ICE_SINGLE_ACT_VSI_FORWARDING |
				ICE_SINGLE_ACT_VALID_BIT;
		break;
	case ICE_FWD_TO_Q:
		act |= ICE_SINGLE_ACT_TO_Q;
		act |= (f_info->fwd_id.q_id << ICE_SINGLE_ACT_Q_INDEX_S) &
			ICE_SINGLE_ACT_Q_INDEX_M;
		break;
	case ICE_DROP_PACKET:
		act |= ICE_SINGLE_ACT_VSI_FORWARDING | ICE_SINGLE_ACT_DROP |
			ICE_SINGLE_ACT_VALID_BIT;
		break;
	case ICE_FWD_TO_QGRP:
		q_rgn = f_info->qgrp_size > 0 ?
			(uint8_t)ice_ilog2(f_info->qgrp_size) : 0;
		act |= ICE_SINGLE_ACT_TO_Q;
		act |= (f_info->fwd_id.q_id << ICE_SINGLE_ACT_Q_INDEX_S) &
			ICE_SINGLE_ACT_Q_INDEX_M;
		act |= (q_rgn << ICE_SINGLE_ACT_Q_REGION_S) &
			ICE_SINGLE_ACT_Q_REGION_M;
		break;
	default:
		return;
	}

	if (f_info->lb_en)
		act |= ICE_SINGLE_ACT_LB_ENABLE;
	if (f_info->lan_en)
		act |= ICE_SINGLE_ACT_LAN_ENABLE;

	switch (f_info->lkup_type) {
	case ICE_SW_LKUP_MAC:
		daddr = f_info->l_data.mac.mac_addr;
		break;
	case ICE_SW_LKUP_VLAN:
		vlan_id = f_info->l_data.vlan.vlan_id;
		if (f_info->l_data.vlan.tpid_valid)
			vlan_tpid = f_info->l_data.vlan.tpid;
		if (f_info->fltr_act == ICE_FWD_TO_VSI ||
		    f_info->fltr_act == ICE_FWD_TO_VSI_LIST) {
			act |= ICE_SINGLE_ACT_PRUNE;
			act |= ICE_SINGLE_ACT_EGRESS | ICE_SINGLE_ACT_INGRESS;
		}
		break;
	case ICE_SW_LKUP_ETHERTYPE_MAC:
		daddr = f_info->l_data.ethertype_mac.mac_addr;
		/* fall-through */
	case ICE_SW_LKUP_ETHERTYPE:
		off = (uint16_t *)(eth_hdr + ICE_ETH_ETHTYPE_OFFSET);
		*off = htobe16(f_info->l_data.ethertype_mac.ethertype);
		break;
	case ICE_SW_LKUP_MAC_VLAN:
		daddr = f_info->l_data.mac_vlan.mac_addr;
		vlan_id = f_info->l_data.mac_vlan.vlan_id;
		break;
	case ICE_SW_LKUP_PROMISC_VLAN:
		vlan_id = f_info->l_data.mac_vlan.vlan_id;
		/* fall-through */
	case ICE_SW_LKUP_PROMISC:
		daddr = f_info->l_data.mac_vlan.mac_addr;
		break;
	default:
		break;
	}

	s_rule->hdr.type = (f_info->flag & ICE_FLTR_RX) ?
		htole16(ICE_AQC_SW_RULES_T_LKUP_RX) :
		htole16(ICE_AQC_SW_RULES_T_LKUP_TX);

	/* Recipe set depending on lookup type */
	s_rule->recipe_id = htole16(f_info->lkup_type);
	s_rule->src = htole16(f_info->src);
	s_rule->act = htole32(act);

	if (daddr)
		memcpy(eth_hdr + ICE_ETH_DA_OFFSET, daddr, ETHER_ADDR_LEN);

	if (!(vlan_id > ICE_MAX_VLAN_ID)) {
		off = (uint16_t *)(eth_hdr + ICE_ETH_VLAN_TCI_OFFSET);
		*off = htobe16(vlan_id);
		off = (uint16_t *)(eth_hdr + ICE_ETH_ETHTYPE_OFFSET);
		*off = htobe16(vlan_tpid);
	}

	/* Create the switch rule with the final dummy Ethernet header */
	if (opc != ice_aqc_opc_update_sw_rules)
		s_rule->hdr_len = htole16(eth_hdr_sz);
}

/**
 * ice_aq_sw_rules - add/update/remove switch rules
 * @hw: pointer to the HW struct
 * @rule_list: pointer to switch rule population list
 * @rule_list_sz: total size of the rule list in bytes
 * @num_rules: number of switch rules in the rule_list
 * @opc: switch rules population command type - pass in the command opcode
 * @cd: pointer to command details structure or NULL
 *
 * Add(0x02a0)/Update(0x02a1)/Remove(0x02a2) switch rules commands to firmware
 */
enum ice_status
ice_aq_sw_rules(struct ice_hw *hw, void *rule_list, uint16_t rule_list_sz,
		uint8_t num_rules, enum ice_adminq_opc opc,
		struct ice_sq_cd *cd)
{
	struct ice_aq_desc desc;
	enum ice_status status;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);

	if (opc != ice_aqc_opc_add_sw_rules &&
	    opc != ice_aqc_opc_update_sw_rules &&
	    opc != ice_aqc_opc_remove_sw_rules)
		return ICE_ERR_PARAM;

	ice_fill_dflt_direct_cmd_desc(&desc, opc);

	desc.flags |= htole16(ICE_AQ_FLAG_RD);
	desc.params.sw_rules.num_rules_fltr_entry_index = htole16(num_rules);
	status = ice_aq_send_cmd(hw, &desc, rule_list, rule_list_sz, cd);
	if (opc != ice_aqc_opc_add_sw_rules &&
	    hw->adminq.sq_last_status == ICE_AQ_RC_ENOENT)
		status = ICE_ERR_DOES_NOT_EXIST;

	return status;
}

/**
 * ice_create_pkt_fwd_rule
 * @hw: pointer to the hardware structure
 * @recp_list: corresponding filter management list
 * @f_entry: entry containing packet forwarding information
 *
 * Create switch rule with given filter information and add an entry
 * to the corresponding filter management list to track this switch rule
 * and VSI mapping
 */
enum ice_status
ice_create_pkt_fwd_rule(struct ice_hw *hw, struct ice_sw_recipe *recp_list,
			struct ice_fltr_list_entry *f_entry)
{
	struct ice_fltr_mgmt_list_entry *fm_entry;
	struct ice_sw_rule_lkup_rx_tx *s_rule;
	enum ice_status status;

	s_rule = (struct ice_sw_rule_lkup_rx_tx *)
		ice_malloc(hw, ice_struct_size(s_rule, hdr_data,
					       ICE_DUMMY_ETH_HDR_LEN));
	if (!s_rule)
		return ICE_ERR_NO_MEMORY;
	fm_entry = (struct ice_fltr_mgmt_list_entry *)
		   ice_malloc(hw, sizeof(*fm_entry));
	if (!fm_entry) {
		status = ICE_ERR_NO_MEMORY;
		goto ice_create_pkt_fwd_rule_exit;
	}

	fm_entry->fltr_info = f_entry->fltr_info;

	/* Initialize all the fields for the management entry */
	fm_entry->vsi_count = 1;
	fm_entry->lg_act_idx = ICE_INVAL_LG_ACT_INDEX;
	fm_entry->sw_marker_id = ICE_INVAL_SW_MARKER_ID;
	fm_entry->counter_index = ICE_INVAL_COUNTER_ID;

	ice_fill_sw_rule(hw, &fm_entry->fltr_info, s_rule,
			 ice_aqc_opc_add_sw_rules);

	status = ice_aq_sw_rules(hw, s_rule,
				 ice_struct_size(s_rule, hdr_data,
						 ICE_DUMMY_ETH_HDR_LEN),
				 1, ice_aqc_opc_add_sw_rules, NULL);
	if (status) {
		ice_free(hw, fm_entry);
		goto ice_create_pkt_fwd_rule_exit;
	}

	f_entry->fltr_info.fltr_rule_id = le16toh(s_rule->index);
	fm_entry->fltr_info.fltr_rule_id = le16toh(s_rule->index);

	/* The book keeping entries will get removed when base driver
	 * calls remove filter AQ command
	 */
	TAILQ_INSERT_HEAD(&recp_list->filt_rules, fm_entry, list_entry);

ice_create_pkt_fwd_rule_exit:
	ice_free(hw, s_rule);
	return status;
}

/**
 * ice_aq_alloc_free_res - command to allocate/free resources
 * @hw: pointer to the HW struct
 * @num_entries: number of resource entries in buffer
 * @buf: Indirect buffer to hold data parameters and response
 * @buf_size: size of buffer for indirect commands
 * @opc: pass in the command opcode
 * @cd: pointer to command details structure or NULL
 *
 * Helper function to allocate/free resources using the admin queue commands
 */
enum ice_status
ice_aq_alloc_free_res(struct ice_hw *hw, uint16_t num_entries,
    struct ice_aqc_alloc_free_res_elem *buf, uint16_t buf_size,
    enum ice_adminq_opc opc, struct ice_sq_cd *cd)
{
	struct ice_aqc_alloc_free_res_cmd *cmd;
	struct ice_aq_desc desc;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);

	cmd = &desc.params.sw_res_ctrl;

	if (!buf)
		return ICE_ERR_PARAM;

	if (buf_size < FLEX_ARRAY_SIZE(buf, elem, num_entries))
		return ICE_ERR_PARAM;

	ice_fill_dflt_direct_cmd_desc(&desc, opc);

	desc.flags |= htole16(ICE_AQ_FLAG_RD);

	cmd->num_entries = htole16(num_entries);

	return ice_aq_send_cmd(hw, &desc, buf, buf_size, cd);
}

/**
 * ice_aq_alloc_free_vsi_list
 * @hw: pointer to the HW struct
 * @vsi_list_id: VSI list ID returned or used for lookup
 * @lkup_type: switch rule filter lookup type
 * @opc: switch rules population command type - pass in the command opcode
 *
 * allocates or free a VSI list resource
 */
enum ice_status
ice_aq_alloc_free_vsi_list(struct ice_hw *hw, uint16_t *vsi_list_id,
			   enum ice_sw_lkup_type lkup_type,
			   enum ice_adminq_opc opc)
{
	struct ice_aqc_alloc_free_res_elem *sw_buf;
	struct ice_aqc_res_elem *vsi_ele;
	enum ice_status status;
	uint16_t buf_len;

	buf_len = ice_struct_size(sw_buf, elem, 1);
	sw_buf = (struct ice_aqc_alloc_free_res_elem *)ice_malloc(hw, buf_len);
	if (!sw_buf)
		return ICE_ERR_NO_MEMORY;
	sw_buf->num_elems = htole16(1);

	if (lkup_type == ICE_SW_LKUP_MAC ||
	    lkup_type == ICE_SW_LKUP_MAC_VLAN ||
	    lkup_type == ICE_SW_LKUP_ETHERTYPE ||
	    lkup_type == ICE_SW_LKUP_ETHERTYPE_MAC ||
	    lkup_type == ICE_SW_LKUP_PROMISC ||
	    lkup_type == ICE_SW_LKUP_PROMISC_VLAN ||
	    lkup_type == ICE_SW_LKUP_DFLT ||
	    lkup_type == ICE_SW_LKUP_LAST) {
		sw_buf->res_type = htole16(ICE_AQC_RES_TYPE_VSI_LIST_REP);
	} else if (lkup_type == ICE_SW_LKUP_VLAN) {
		sw_buf->res_type =
			htole16(ICE_AQC_RES_TYPE_VSI_LIST_PRUNE);
	} else {
		status = ICE_ERR_PARAM;
		goto ice_aq_alloc_free_vsi_list_exit;
	}

	if (opc == ice_aqc_opc_free_res)
		sw_buf->elem[0].e.sw_resp = htole16(*vsi_list_id);

	status = ice_aq_alloc_free_res(hw, 1, sw_buf, buf_len, opc, NULL);
	if (status)
		goto ice_aq_alloc_free_vsi_list_exit;

	if (opc == ice_aqc_opc_alloc_res) {
		vsi_ele = &sw_buf->elem[0];
		*vsi_list_id = le16toh(vsi_ele->e.sw_resp);
	}

ice_aq_alloc_free_vsi_list_exit:
	ice_free(hw, sw_buf);
	return status;
}

/**
 * ice_update_vsi_list_rule
 * @hw: pointer to the hardware structure
 * @vsi_handle_arr: array of VSI handles to form a VSI list
 * @num_vsi: number of VSI handles in the array
 * @vsi_list_id: VSI list ID generated as part of allocate resource
 * @remove: Boolean value to indicate if this is a remove action
 * @opc: switch rules population command type - pass in the command opcode
 * @lkup_type: lookup type of the filter
 *
 * Call AQ command to add a new switch rule or update existing switch rule
 * using the given VSI list ID
 */
enum ice_status
ice_update_vsi_list_rule(struct ice_hw *hw, uint16_t *vsi_handle_arr,
    uint16_t num_vsi, uint16_t vsi_list_id, bool remove,
    enum ice_adminq_opc opc, enum ice_sw_lkup_type lkup_type)
{
	struct ice_sw_rule_vsi_list *s_rule;
	enum ice_status status;
	uint16_t s_rule_size;
	uint16_t rule_type;
	int i;

	if (!num_vsi)
		return ICE_ERR_PARAM;

	if (lkup_type == ICE_SW_LKUP_MAC ||
	    lkup_type == ICE_SW_LKUP_MAC_VLAN ||
	    lkup_type == ICE_SW_LKUP_ETHERTYPE ||
	    lkup_type == ICE_SW_LKUP_ETHERTYPE_MAC ||
	    lkup_type == ICE_SW_LKUP_PROMISC ||
	    lkup_type == ICE_SW_LKUP_PROMISC_VLAN ||
	    lkup_type == ICE_SW_LKUP_DFLT ||
	    lkup_type == ICE_SW_LKUP_LAST)
		rule_type = remove ? ICE_AQC_SW_RULES_T_VSI_LIST_CLEAR :
			ICE_AQC_SW_RULES_T_VSI_LIST_SET;
	else if (lkup_type == ICE_SW_LKUP_VLAN)
		rule_type = remove ? ICE_AQC_SW_RULES_T_PRUNE_LIST_CLEAR :
			ICE_AQC_SW_RULES_T_PRUNE_LIST_SET;
	else
		return ICE_ERR_PARAM;

	s_rule_size = (uint16_t)ice_struct_size(s_rule, vsi, num_vsi);
	s_rule = (struct ice_sw_rule_vsi_list *)ice_malloc(hw, s_rule_size);
	if (!s_rule)
		return ICE_ERR_NO_MEMORY;
	for (i = 0; i < num_vsi; i++) {
		if (!ice_is_vsi_valid(hw, vsi_handle_arr[i])) {
			status = ICE_ERR_PARAM;
			goto exit;
		}
		/* AQ call requires hw_vsi_id(s) */
		s_rule->vsi[i] =
			htole16(hw->vsi_ctx[vsi_handle_arr[i]]->vsi_num);
	}

	s_rule->hdr.type = htole16(rule_type);
	s_rule->number_vsi = htole16(num_vsi);
	s_rule->index = htole16(vsi_list_id);

	status = ice_aq_sw_rules(hw, s_rule, s_rule_size, 1, opc, NULL);

exit:
	ice_free(hw, s_rule);
	return status;
}

/**
 * ice_create_vsi_list_rule - Creates and populates a VSI list rule
 * @hw: pointer to the HW struct
 * @vsi_handle_arr: array of VSI handles to form a VSI list
 * @num_vsi: number of VSI handles in the array
 * @vsi_list_id: stores the ID of the VSI list to be created
 * @lkup_type: switch rule filter's lookup type
 */
enum ice_status
ice_create_vsi_list_rule(struct ice_hw *hw, uint16_t *vsi_handle_arr,
     uint16_t num_vsi, uint16_t *vsi_list_id, enum ice_sw_lkup_type lkup_type)
{
	enum ice_status status;

	status = ice_aq_alloc_free_vsi_list(hw, vsi_list_id, lkup_type,
					    ice_aqc_opc_alloc_res);
	if (status)
		return status;

	/* Update the newly created VSI list to include the specified VSIs */
	return ice_update_vsi_list_rule(hw, vsi_handle_arr, num_vsi,
					*vsi_list_id, false,
					ice_aqc_opc_add_sw_rules, lkup_type);
}

/**
 * ice_update_pkt_fwd_rule
 * @hw: pointer to the hardware structure
 * @f_info: filter information for switch rule
 *
 * Call AQ command to update a previously created switch rule with a
 * VSI list ID
 */
enum ice_status
ice_update_pkt_fwd_rule(struct ice_hw *hw, struct ice_fltr_info *f_info)
{
	struct ice_sw_rule_lkup_rx_tx *s_rule;
	enum ice_status status;

	s_rule = (struct ice_sw_rule_lkup_rx_tx *)
		ice_malloc(hw, ice_struct_size(s_rule, hdr_data,
					       ICE_DUMMY_ETH_HDR_LEN));
	if (!s_rule)
		return ICE_ERR_NO_MEMORY;

	ice_fill_sw_rule(hw, f_info, s_rule, ice_aqc_opc_update_sw_rules);

	s_rule->index = htole16(f_info->fltr_rule_id);

	/* Update switch rule with new rule set to forward VSI list */
	status = ice_aq_sw_rules(hw, s_rule,
				 ice_struct_size(s_rule, hdr_data,
						 ICE_DUMMY_ETH_HDR_LEN),
				 1, ice_aqc_opc_update_sw_rules, NULL);

	ice_free(hw, s_rule);
	return status;
}

/**
 * ice_create_vsi_list_map
 * @hw: pointer to the hardware structure
 * @vsi_handle_arr: array of VSI handles to set in the VSI mapping
 * @num_vsi: number of VSI handles in the array
 * @vsi_list_id: VSI list ID generated as part of allocate resource
 *
 * Helper function to create a new entry of VSI list ID to VSI mapping
 * using the given VSI list ID
 */
struct ice_vsi_list_map_info *
ice_create_vsi_list_map(struct ice_hw *hw, uint16_t *vsi_handle_arr,
    uint16_t num_vsi, uint16_t vsi_list_id)
{
	struct ice_switch_info *sw = hw->switch_info;
	struct ice_vsi_list_map_info *v_map;
	int i;

	v_map = (struct ice_vsi_list_map_info *)ice_malloc(hw, sizeof(*v_map));
	if (!v_map)
		return NULL;

	v_map->vsi_list_id = vsi_list_id;
	v_map->ref_cnt = 1;
	for (i = 0; i < num_vsi; i++)
		ice_set_bit(vsi_handle_arr[i], v_map->vsi_map);

	TAILQ_INSERT_HEAD(&sw->vsi_list_map_head, v_map, list_entry);
	return v_map;
}

/**
 * ice_add_marker_act
 * @hw: pointer to the hardware structure
 * @m_ent: the management entry for which sw marker needs to be added
 * @sw_marker: sw marker to tag the Rx descriptor with
 * @l_id: large action resource ID
 *
 * Create a large action to hold software marker and update the switch rule
 * entry pointed by m_ent with newly created large action
 */
enum ice_status
ice_add_marker_act(struct ice_hw *hw, struct ice_fltr_mgmt_list_entry *m_ent,
		   uint16_t sw_marker, uint16_t l_id)
{
	struct ice_sw_rule_lkup_rx_tx *rx_tx;
	struct ice_sw_rule_lg_act *lg_act;
	/* For software marker we need 3 large actions
	 * 1. FWD action: FWD TO VSI or VSI LIST
	 * 2. GENERIC VALUE action to hold the profile ID
	 * 3. GENERIC VALUE action to hold the software marker ID
	 */
	const uint16_t num_lg_acts = 3;
	enum ice_status status;
	uint16_t lg_act_size;
	uint16_t rules_size;
	uint32_t act;
	uint16_t id;

	if (m_ent->fltr_info.lkup_type != ICE_SW_LKUP_MAC)
		return ICE_ERR_PARAM;

	/* Create two back-to-back switch rules and submit them to the HW using
	 * one memory buffer:
	 *    1. Large Action
	 *    2. Look up Tx Rx
	 */
	lg_act_size = (uint16_t)ice_struct_size(lg_act, act, num_lg_acts);
	rules_size = lg_act_size +
		     ice_struct_size(rx_tx, hdr_data, ICE_DUMMY_ETH_HDR_LEN);
	lg_act = (struct ice_sw_rule_lg_act *)ice_malloc(hw, rules_size);
	if (!lg_act)
		return ICE_ERR_NO_MEMORY;

	rx_tx = (struct ice_sw_rule_lkup_rx_tx *)((uint8_t *)lg_act +
	    lg_act_size);

	/* Fill in the first switch rule i.e. large action */
	lg_act->hdr.type = htole16(ICE_AQC_SW_RULES_T_LG_ACT);
	lg_act->index = htole16(l_id);
	lg_act->size = htole16(num_lg_acts);

	/* First action VSI forwarding or VSI list forwarding depending on how
	 * many VSIs
	 */
	id = (m_ent->vsi_count > 1) ? m_ent->fltr_info.fwd_id.vsi_list_id :
		m_ent->fltr_info.fwd_id.hw_vsi_id;

	act = ICE_LG_ACT_VSI_FORWARDING | ICE_LG_ACT_VALID_BIT;
	act |= (id << ICE_LG_ACT_VSI_LIST_ID_S) & ICE_LG_ACT_VSI_LIST_ID_M;
	if (m_ent->vsi_count > 1)
		act |= ICE_LG_ACT_VSI_LIST;
	lg_act->act[0] = htole32(act);

	/* Second action descriptor type */
	act = ICE_LG_ACT_GENERIC;

	act |= (1 << ICE_LG_ACT_GENERIC_VALUE_S) & ICE_LG_ACT_GENERIC_VALUE_M;
	lg_act->act[1] = htole32(act);

	act = (ICE_LG_ACT_GENERIC_OFF_RX_DESC_PROF_IDX <<
	       ICE_LG_ACT_GENERIC_OFFSET_S) & ICE_LG_ACT_GENERIC_OFFSET_M;

	/* Third action Marker value */
	act |= ICE_LG_ACT_GENERIC;
	act |= (sw_marker << ICE_LG_ACT_GENERIC_VALUE_S) &
		ICE_LG_ACT_GENERIC_VALUE_M;

	lg_act->act[2] = htole32(act);

	/* call the fill switch rule to fill the lookup Tx Rx structure */
	ice_fill_sw_rule(hw, &m_ent->fltr_info, rx_tx,
			 ice_aqc_opc_update_sw_rules);

	/* Update the action to point to the large action ID */
	rx_tx->act = htole32(ICE_SINGLE_ACT_PTR |
	    ((l_id << ICE_SINGLE_ACT_PTR_VAL_S) & ICE_SINGLE_ACT_PTR_VAL_M));

	/* Use the filter rule ID of the previously created rule with single
	 * act. Once the update happens, hardware will treat this as large
	 * action
	 */
	rx_tx->index = htole16(m_ent->fltr_info.fltr_rule_id);

	status = ice_aq_sw_rules(hw, lg_act, rules_size, 2,
				 ice_aqc_opc_update_sw_rules, NULL);
	if (!status) {
		m_ent->lg_act_idx = l_id;
		m_ent->sw_marker_id = sw_marker;
	}

	ice_free(hw, lg_act);
	return status;
}

/**
 * ice_add_update_vsi_list
 * @hw: pointer to the hardware structure
 * @m_entry: pointer to current filter management list entry
 * @cur_fltr: filter information from the book keeping entry
 * @new_fltr: filter information with the new VSI to be added
 *
 * Call AQ command to add or update previously created VSI list with new VSI.
 *
 * Helper function to do book keeping associated with adding filter information
 * The algorithm to do the book keeping is described below :
 * When a VSI needs to subscribe to a given filter (MAC/VLAN/Ethtype etc.)
 *	if only one VSI has been added till now
 *		Allocate a new VSI list and add two VSIs
 *		to this list using switch rule command
 *		Update the previously created switch rule with the
 *		newly created VSI list ID
 *	if a VSI list was previously created
 *		Add the new VSI to the previously created VSI list set
 *		using the update switch rule command
 */
enum ice_status
ice_add_update_vsi_list(struct ice_hw *hw,
			struct ice_fltr_mgmt_list_entry *m_entry,
			struct ice_fltr_info *cur_fltr,
			struct ice_fltr_info *new_fltr)
{
	enum ice_status status = ICE_SUCCESS;
	uint16_t vsi_list_id = 0;

	if ((cur_fltr->fltr_act == ICE_FWD_TO_Q ||
	     cur_fltr->fltr_act == ICE_FWD_TO_QGRP))
		return ICE_ERR_NOT_IMPL;

	if ((new_fltr->fltr_act == ICE_FWD_TO_Q ||
	     new_fltr->fltr_act == ICE_FWD_TO_QGRP) &&
	    (cur_fltr->fltr_act == ICE_FWD_TO_VSI ||
	     cur_fltr->fltr_act == ICE_FWD_TO_VSI_LIST))
		return ICE_ERR_NOT_IMPL;

	if (m_entry->vsi_count < 2 && !m_entry->vsi_list_info) {
		/* Only one entry existed in the mapping and it was not already
		 * a part of a VSI list. So, create a VSI list with the old and
		 * new VSIs.
		 */
		struct ice_fltr_info tmp_fltr;
		uint16_t vsi_handle_arr[2];

		/* A rule already exists with the new VSI being added */
		if (cur_fltr->fwd_id.hw_vsi_id == new_fltr->fwd_id.hw_vsi_id)
			return ICE_ERR_ALREADY_EXISTS;

		vsi_handle_arr[0] = cur_fltr->vsi_handle;
		vsi_handle_arr[1] = new_fltr->vsi_handle;
		status = ice_create_vsi_list_rule(hw, &vsi_handle_arr[0], 2,
						  &vsi_list_id,
						  new_fltr->lkup_type);
		if (status)
			return status;

		tmp_fltr = *new_fltr;
		tmp_fltr.fltr_rule_id = cur_fltr->fltr_rule_id;
		tmp_fltr.fltr_act = ICE_FWD_TO_VSI_LIST;
		tmp_fltr.fwd_id.vsi_list_id = vsi_list_id;
		/* Update the previous switch rule of "MAC forward to VSI" to
		 * "MAC fwd to VSI list"
		 */
		status = ice_update_pkt_fwd_rule(hw, &tmp_fltr);
		if (status)
			return status;

		cur_fltr->fwd_id.vsi_list_id = vsi_list_id;
		cur_fltr->fltr_act = ICE_FWD_TO_VSI_LIST;
		m_entry->vsi_list_info =
			ice_create_vsi_list_map(hw, &vsi_handle_arr[0], 2,
						vsi_list_id);

		if (!m_entry->vsi_list_info)
			return ICE_ERR_NO_MEMORY;

		/* If this entry was large action then the large action needs
		 * to be updated to point to FWD to VSI list
		 */
		if (m_entry->sw_marker_id != ICE_INVAL_SW_MARKER_ID)
			status =
			    ice_add_marker_act(hw, m_entry,
					       m_entry->sw_marker_id,
					       m_entry->lg_act_idx);
	} else {
		uint16_t vsi_handle = new_fltr->vsi_handle;
		enum ice_adminq_opc opcode;

		if (!m_entry->vsi_list_info)
			return ICE_ERR_CFG;

		/* A rule already exists with the new VSI being added */
		if (ice_is_bit_set(m_entry->vsi_list_info->vsi_map, vsi_handle))
			return ICE_SUCCESS;

		/* Update the previously created VSI list set with
		 * the new VSI ID passed in
		 */
		vsi_list_id = cur_fltr->fwd_id.vsi_list_id;
		opcode = ice_aqc_opc_update_sw_rules;

		status = ice_update_vsi_list_rule(hw, &vsi_handle, 1,
						  vsi_list_id, false, opcode,
						  new_fltr->lkup_type);
		/* update VSI list mapping info with new VSI ID */
		if (!status)
			ice_set_bit(vsi_handle,
				    m_entry->vsi_list_info->vsi_map);
	}
	if (!status)
		m_entry->vsi_count++;
	return status;
}

/**
 * ice_add_rule_internal - add rule for a given lookup type
 * @hw: pointer to the hardware structure
 * @recp_list: recipe list for which rule has to be added
 * @lport: logic port number on which function add rule
 * @f_entry: structure containing MAC forwarding information
 *
 * Adds or updates the rule lists for a given recipe
 */
enum ice_status
ice_add_rule_internal(struct ice_hw *hw, struct ice_sw_recipe *recp_list,
		      uint8_t lport, struct ice_fltr_list_entry *f_entry)
{
	struct ice_fltr_info *new_fltr, *cur_fltr;
	struct ice_fltr_mgmt_list_entry *m_entry;
	struct ice_lock *rule_lock; /* Lock to protect filter rule list */
	enum ice_status status = ICE_SUCCESS;

	if (!ice_is_vsi_valid(hw, f_entry->fltr_info.vsi_handle))
		return ICE_ERR_PARAM;

	/* Load the hw_vsi_id only if the fwd action is fwd to VSI */
	if (f_entry->fltr_info.fltr_act == ICE_FWD_TO_VSI)
		f_entry->fltr_info.fwd_id.hw_vsi_id =
			hw->vsi_ctx[f_entry->fltr_info.vsi_handle]->vsi_num;

	rule_lock = &recp_list->filt_rule_lock;
#if 0
	ice_acquire_lock(rule_lock);
#endif
	new_fltr = &f_entry->fltr_info;
	if (new_fltr->flag & ICE_FLTR_RX)
		new_fltr->src = lport;
	else if (new_fltr->flag & (ICE_FLTR_TX | ICE_FLTR_RX_LB))
		new_fltr->src =
			hw->vsi_ctx[f_entry->fltr_info.vsi_handle]->vsi_num;

	m_entry = ice_find_rule_entry(&recp_list->filt_rules, new_fltr);
	if (!m_entry) {
		status = ice_create_pkt_fwd_rule(hw, recp_list, f_entry);
		goto exit_add_rule_internal;
	}

	cur_fltr = &m_entry->fltr_info;
	status = ice_add_update_vsi_list(hw, m_entry, cur_fltr, new_fltr);

exit_add_rule_internal:
#if 0
	ice_release_lock(rule_lock);
#endif
	return status;
}

/**
 * ice_add_mac_rule - Add a MAC address based filter rule
 * @hw: pointer to the hardware structure
 * @list: list of MAC addresses and forwarding information
 * @sw: pointer to switch info struct for which function add rule
 * @lport: logic port number on which function add rule
 *
 * IMPORTANT: When the umac_shared flag is set to false and 'list' has
 * multiple unicast addresses, the function assumes that all the
 * addresses are unique in a given add_mac call. It doesn't
 * check for duplicates in this case, removing duplicates from a given
 * list should be taken care of in the caller of this function.
 */
enum ice_status
ice_add_mac_rule(struct ice_hw *hw, struct ice_fltr_list_head *list,
		 struct ice_switch_info *sw, uint8_t lport)
{
	struct ice_sw_recipe *recp_list = &sw->recp_list[ICE_SW_LKUP_MAC];
	struct ice_sw_rule_lkup_rx_tx *s_rule, *r_iter;
	struct ice_fltr_list_entry *list_itr;
	struct ice_fltr_mgmt_list_head *rule_head;
	uint16_t total_elem_left, s_rule_size;
#if 0
	struct ice_lock *rule_lock; /* Lock to protect filter rule list */
#endif
	enum ice_status status = ICE_SUCCESS;
	uint16_t num_unicast = 0;
	uint8_t elem_sent;

	s_rule = NULL;
#if 0
	rule_lock = &recp_list->filt_rule_lock;
#endif
	rule_head = &recp_list->filt_rules;

	TAILQ_FOREACH(list_itr, list, list_entry) {
		uint8_t *add = &list_itr->fltr_info.l_data.mac.mac_addr[0];
		uint16_t vsi_handle;
		uint16_t hw_vsi_id;

		list_itr->fltr_info.flag = ICE_FLTR_TX;
		vsi_handle = list_itr->fltr_info.vsi_handle;
		if (!ice_is_vsi_valid(hw, vsi_handle))
			return ICE_ERR_PARAM;
		hw_vsi_id = hw->vsi_ctx[vsi_handle]->vsi_num;
		if (list_itr->fltr_info.fltr_act == ICE_FWD_TO_VSI)
			list_itr->fltr_info.fwd_id.hw_vsi_id = hw_vsi_id;
		/* update the src in case it is VSI num */
		if (list_itr->fltr_info.src_id != ICE_SRC_ID_VSI)
			return ICE_ERR_PARAM;
		list_itr->fltr_info.src = hw_vsi_id;
		if (list_itr->fltr_info.lkup_type != ICE_SW_LKUP_MAC ||
		    ETHER_IS_ANYADDR(add))
			return ICE_ERR_PARAM;
		if (!ETHER_IS_MULTICAST(add) && !hw->umac_shared) {
			/* Don't overwrite the unicast address */
#if 0
			ice_acquire_lock(rule_lock);
#endif
			if (ice_find_rule_entry(rule_head,
						&list_itr->fltr_info)) {
#if 0
				ice_release_lock(rule_lock);
#endif
				continue;
			}
#if 0
			ice_release_lock(rule_lock);
#endif
			num_unicast++;
		} else if (ETHER_IS_MULTICAST(add) || hw->umac_shared) {
			list_itr->status =
				ice_add_rule_internal(hw, recp_list, lport,
						      list_itr);
			if (list_itr->status)
				return list_itr->status;
		}
	}
#if 0
	ice_acquire_lock(rule_lock);
#endif
	/* Exit if no suitable entries were found for adding bulk switch rule */
	if (!num_unicast) {
		status = ICE_SUCCESS;
		goto ice_add_mac_exit;
	}

	/* Allocate switch rule buffer for the bulk update for unicast */
	s_rule_size = ice_struct_size(s_rule, hdr_data, ICE_DUMMY_ETH_HDR_LEN);
	s_rule = (struct ice_sw_rule_lkup_rx_tx *)
		ice_calloc(hw, num_unicast, s_rule_size);
	if (!s_rule) {
		status = ICE_ERR_NO_MEMORY;
		goto ice_add_mac_exit;
	}

	r_iter = s_rule;
	TAILQ_FOREACH(list_itr, list, list_entry) {
		struct ice_fltr_info *f_info = &list_itr->fltr_info;
		uint8_t *mac_addr = &f_info->l_data.mac.mac_addr[0];

		if (!ETHER_IS_MULTICAST(mac_addr)) {
			ice_fill_sw_rule(hw, &list_itr->fltr_info, r_iter,
					 ice_aqc_opc_add_sw_rules);
			r_iter = (struct ice_sw_rule_lkup_rx_tx *)
				((uint8_t *)r_iter + s_rule_size);
		}
	}

	/* Call AQ bulk switch rule update for all unicast addresses */
	r_iter = s_rule;
	/* Call AQ switch rule in AQ_MAX chunk */
	for (total_elem_left = num_unicast; total_elem_left > 0;
	     total_elem_left -= elem_sent) {
		struct ice_sw_rule_lkup_rx_tx *entry = r_iter;

		elem_sent = MIN(total_elem_left,
		    (ICE_AQ_MAX_BUF_LEN / s_rule_size));
		status = ice_aq_sw_rules(hw, entry, elem_sent * s_rule_size,
					 elem_sent, ice_aqc_opc_add_sw_rules,
					 NULL);
		if (status)
			goto ice_add_mac_exit;
		r_iter = (struct ice_sw_rule_lkup_rx_tx *)
			((uint8_t *)r_iter + (elem_sent * s_rule_size));
	}

	/* Fill up rule ID based on the value returned from FW */
	r_iter = s_rule;
	TAILQ_FOREACH(list_itr, list, list_entry) {
		struct ice_fltr_info *f_info = &list_itr->fltr_info;
		uint8_t *mac_addr = &f_info->l_data.mac.mac_addr[0];
		struct ice_fltr_mgmt_list_entry *fm_entry;

		if (!ETHER_IS_MULTICAST(mac_addr)) {
			f_info->fltr_rule_id = le16toh(r_iter->index);
			f_info->fltr_act = ICE_FWD_TO_VSI;
			/* Create an entry to track this MAC address */
			fm_entry = (struct ice_fltr_mgmt_list_entry *)
				ice_malloc(hw, sizeof(*fm_entry));
			if (!fm_entry) {
				status = ICE_ERR_NO_MEMORY;
				goto ice_add_mac_exit;
			}
			fm_entry->fltr_info = *f_info;
			fm_entry->vsi_count = 1;
			/* The book keeping entries will get removed when
			 * base driver calls remove filter AQ command
			 */

			TAILQ_INSERT_HEAD(rule_head, fm_entry, list_entry);
			r_iter = (struct ice_sw_rule_lkup_rx_tx *)
				((uint8_t *)r_iter + s_rule_size);
		}
	}

ice_add_mac_exit:
#if 0
	ice_release_lock(rule_lock);
#endif
	if (s_rule)
		ice_free(hw, s_rule);
	return status;
}

/**
 * ice_add_mac - Add a MAC address based filter rule
 * @hw: pointer to the hardware structure
 * @list: list of MAC addresses and forwarding information
 *
 * Function add MAC rule for logical port from HW struct
 */
enum ice_status
ice_add_mac(struct ice_hw *hw, struct ice_fltr_list_head *list)
{
	if (!list || !hw)
		return ICE_ERR_PARAM;

	return ice_add_mac_rule(hw, list, hw->switch_info,
	    hw->port_info->lport);
}

/**
 * ice_add_vsi_mac_filter - Add a MAC address filter for a VSI
 * @vsi: the VSI to add the filter for
 * @addr: MAC address to add a filter for
 *
 * Add a MAC address filter for a given VSI. This is a wrapper around
 * ice_add_mac to simplify the interface. First, it only accepts a single
 * address, so we don't have to mess around with the list setup in other
 * functions. Second, it ignores the ICE_ERR_ALREADY_EXISTS error, so that
 * callers don't need to worry about attempting to add the same filter twice.
 */
int
ice_add_vsi_mac_filter(struct ice_vsi *vsi, uint8_t *addr)
{
	struct ice_softc *sc = vsi->sc;
	struct ice_fltr_list_head mac_addr_list;
	struct ice_hw *hw = &vsi->sc->hw;
	enum ice_status status;
	int err = 0;

	TAILQ_INIT(&mac_addr_list);

	err = ice_add_mac_to_list(vsi, &mac_addr_list, addr, ICE_FWD_TO_VSI);
	if (err)
		goto free_mac_list;

	status = ice_add_mac(hw, &mac_addr_list);
	if (status != ICE_ERR_ALREADY_EXISTS && status) {
		printf("%s: Failed to add a filter for MAC %s, "
		    "err %s aq_err %s\n", sc->sc_dev.dv_xname,
		    ether_sprintf(addr), ice_status_str(status),
		    ice_aq_str(hw->adminq.sq_last_status));
		err = (EIO);
	}

free_mac_list:
	ice_free_fltr_list(&mac_addr_list);
	return err;
}

/**
 * ice_cfg_pf_default_mac_filters - Setup default unicast and broadcast addrs
 * @sc: device softc structure
 *
 * Program the default unicast and broadcast filters for the PF VSI.
 */
int
ice_cfg_pf_default_mac_filters(struct ice_softc *sc)
{
	struct ice_hw *hw = &sc->hw;
	struct ice_vsi *vsi = &sc->pf_vsi;
	int err;

	/* Add the LAN MAC address */
	err = ice_add_vsi_mac_filter(vsi, hw->port_info->mac.lan_addr);
	if (err)
		return err;

	/* Add the broadcast address */
	err = ice_add_vsi_mac_filter(vsi, etherbroadcastaddr);
	if (err)
		return err;

	return (0);
}

/**
 * ice_init_tx_tracking - Initialize Tx queue software tracking values
 * @vsi: the VSI to initialize
 *
 * Initialize Tx queue software tracking values, including the Report Status
 * queue, and related software tracking values.
 */
void
ice_init_tx_tracking(struct ice_vsi *vsi)
{
	struct ice_tx_queue *txq;
	size_t j;
	int i;

	for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) {

		txq->tx_rs_cidx = txq->tx_rs_pidx = 0;

		/* Initialize the last processed descriptor to be the end of
		 * the ring, rather than the start, so that we avoid an
		 * off-by-one error in ice_ift_txd_credits_update for the
		 * first packet.
		 */
		txq->tx_cidx_processed = txq->desc_count - 1;

		for (j = 0; j < txq->desc_count; j++)
			txq->tx_rsq[j] = ICE_QIDX_INVALID;
	}
}

/**
 * ice_setup_tx_ctx - Setup an ice_tlan_ctx structure for a queue
 * @txq: the Tx queue to configure
 * @tlan_ctx: the Tx LAN queue context structure to initialize
 * @pf_q: real queue number
 */
int
ice_setup_tx_ctx(struct ice_tx_queue *txq, struct ice_tlan_ctx *tlan_ctx,
    uint16_t pf_q)
{
	struct ice_vsi *vsi = txq->vsi;
	struct ice_softc *sc = vsi->sc;
	struct ice_hw *hw = &sc->hw;

	tlan_ctx->port_num = hw->port_info->lport;

	/* number of descriptors in the queue */
	tlan_ctx->qlen = txq->desc_count;

	/* set the transmit queue base address, defined in 128 byte units */
	tlan_ctx->base = txq->tx_paddr >> 7;

	tlan_ctx->pf_num = hw->pf_id;

	switch (vsi->type) {
	case ICE_VSI_PF:
		tlan_ctx->vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_PF;
		break;
	case ICE_VSI_VMDQ2:
		tlan_ctx->vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_VMQ;
		break;
	default:
		return (ENODEV);
	}

	tlan_ctx->src_vsi = hw->vsi_ctx[vsi->idx]->vsi_num;
#if 0
	/* Enable TSO */
	tlan_ctx->tso_ena = 1;
	tlan_ctx->internal_usage_flag = 1;

	tlan_ctx->tso_qnum = pf_q;
#endif
	/*
	 * Stick with the older legacy Tx queue interface, instead of the new
	 * advanced queue interface.
	 */
	tlan_ctx->legacy_int = 1;

	/* Descriptor WB mode */
	tlan_ctx->wb_mode = 0;

	return (0);
}

/**
 * ice_get_vsi_ctx - return the VSI context entry for a given VSI handle
 * @hw: pointer to the HW struct
 * @vsi_handle: VSI handle
 *
 * return the VSI context entry for a given VSI handle
 */
struct ice_vsi_ctx *
ice_get_vsi_ctx(struct ice_hw *hw, uint16_t vsi_handle)
{
	return (vsi_handle >= ICE_MAX_VSI) ? NULL : hw->vsi_ctx[vsi_handle];
}

/**
 * ice_get_lan_q_ctx - get the LAN queue context for the given VSI and TC
 * @hw: pointer to the HW struct
 * @vsi_handle: software VSI handle
 * @tc: TC number
 * @q_handle: software queue handle
 */
struct ice_q_ctx *
ice_get_lan_q_ctx(struct ice_hw *hw, uint16_t vsi_handle, uint8_t tc,
    uint16_t q_handle)
{
	struct ice_vsi_ctx *vsi;
	struct ice_q_ctx *q_ctx;

	vsi = ice_get_vsi_ctx(hw, vsi_handle);
	if (!vsi)
		return NULL;
	if (q_handle >= vsi->num_lan_q_entries[tc])
		return NULL;
	if (!vsi->lan_q_ctx[tc])
		return NULL;
	q_ctx = vsi->lan_q_ctx[tc];
	return &q_ctx[q_handle];
}

/**
 * ice_sched_find_node_in_subtree - Find node in part of base node subtree
 * @hw: pointer to the HW struct
 * @base: pointer to the base node
 * @node: pointer to the node to search
 *
 * This function checks whether a given node is part of the base node
 * subtree or not
 */
bool
ice_sched_find_node_in_subtree(struct ice_hw *hw, struct ice_sched_node *base,
			       struct ice_sched_node *node)
{
	uint8_t i;

	for (i = 0; i < base->num_children; i++) {
		struct ice_sched_node *child = base->children[i];

		if (node == child)
			return true;

		if (child->tx_sched_layer > node->tx_sched_layer)
			return false;

		/* this recursion is intentional, and wouldn't
		 * go more than 8 calls
		 */
		if (ice_sched_find_node_in_subtree(hw, child, node))
			return true;
	}
	return false;
}

/**
 * ice_sched_get_free_qgrp - Scan all queue group siblings and find a free node
 * @pi: port information structure
 * @vsi_node: software VSI handle
 * @qgrp_node: first queue group node identified for scanning
 * @owner: LAN or RDMA
 *
 * This function retrieves a free LAN or RDMA queue group node by scanning
 * qgrp_node and its siblings for the queue group with the fewest number
 * of queues currently assigned.
 */
struct ice_sched_node *
ice_sched_get_free_qgrp(struct ice_port_info *pi,
			struct ice_sched_node *vsi_node,
			struct ice_sched_node *qgrp_node, uint8_t owner)
{
	struct ice_sched_node *min_qgrp;
	uint8_t min_children;

	if (!qgrp_node)
		return qgrp_node;
	min_children = qgrp_node->num_children;
	if (!min_children)
		return qgrp_node;
	min_qgrp = qgrp_node;
	/* scan all queue groups until find a node which has less than the
	 * minimum number of children. This way all queue group nodes get
	 * equal number of shares and active. The bandwidth will be equally
	 * distributed across all queues.
	 */
	while (qgrp_node) {
		/* make sure the qgroup node is part of the VSI subtree */
		if (ice_sched_find_node_in_subtree(pi->hw, vsi_node, qgrp_node))
			if (qgrp_node->num_children < min_children &&
			    qgrp_node->owner == owner) {
				/* replace the new min queue group node */
				min_qgrp = qgrp_node;
				min_children = min_qgrp->num_children;
				/* break if it has no children, */
				if (!min_children)
					break;
			}
		qgrp_node = qgrp_node->sibling;
	}
	return min_qgrp;
}

/**
 * ice_sched_get_qgrp_layer - get the current queue group layer number
 * @hw: pointer to the HW struct
 *
 * This function returns the current queue group layer number
 */
uint8_t
ice_sched_get_qgrp_layer(struct ice_hw *hw)
{
	/* It's always total layers - 1, the array is 0 relative so -2 */
	return hw->num_tx_sched_layers - ICE_QGRP_LAYER_OFFSET;
}

/**
 * ice_sched_get_vsi_layer - get the current VSI layer number
 * @hw: pointer to the HW struct
 *
 * This function returns the current VSI layer number
 */
uint8_t
ice_sched_get_vsi_layer(struct ice_hw *hw)
{
	/* Num Layers       VSI layer
	 *     9               6
	 *     7               4
	 *     5 or less       sw_entry_point_layer
	 */
	/* calculate the VSI layer based on number of layers. */
	if (hw->num_tx_sched_layers == ICE_SCHED_9_LAYERS)
		return hw->num_tx_sched_layers - ICE_VSI_LAYER_OFFSET;
	else if (hw->num_tx_sched_layers == ICE_SCHED_5_LAYERS)
		/* qgroup and VSI layers are same */
		return hw->num_tx_sched_layers - ICE_QGRP_LAYER_OFFSET;
	return hw->sw_entry_point_layer;
}

/**
 * ice_sched_get_free_qparent - Get a free LAN or RDMA queue group node
 * @pi: port information structure
 * @vsi_handle: software VSI handle
 * @tc: branch number
 * @owner: LAN or RDMA
 *
 * This function retrieves a free LAN or RDMA queue group node
 */
struct ice_sched_node *
ice_sched_get_free_qparent(struct ice_port_info *pi, uint16_t vsi_handle,
    uint8_t tc, uint8_t owner)
{
	struct ice_sched_node *vsi_node, *qgrp_node;
	struct ice_vsi_ctx *vsi_ctx;
	uint8_t qgrp_layer, vsi_layer;
	uint16_t max_children;

	qgrp_layer = ice_sched_get_qgrp_layer(pi->hw);
	vsi_layer = ice_sched_get_vsi_layer(pi->hw);
	max_children = pi->hw->max_children[qgrp_layer];

	vsi_ctx = ice_get_vsi_ctx(pi->hw, vsi_handle);
	if (!vsi_ctx)
		return NULL;
	vsi_node = vsi_ctx->sched.vsi_node[tc];
	/* validate invalid VSI ID */
	if (!vsi_node)
		return NULL;

	/* If the queue group and vsi layer are same then queues
	 * are all attached directly to VSI
	 */
	if (qgrp_layer == vsi_layer)
		return vsi_node;

	/* get the first queue group node from VSI sub-tree */
	qgrp_node = ice_sched_get_first_node(pi, vsi_node, qgrp_layer);
	while (qgrp_node) {
		/* make sure the qgroup node is part of the VSI subtree */
		if (ice_sched_find_node_in_subtree(pi->hw, vsi_node, qgrp_node))
			if (qgrp_node->num_children < max_children &&
			    qgrp_node->owner == owner)
				break;
		qgrp_node = qgrp_node->sibling;
	}

	/* Select the best queue group */
	return ice_sched_get_free_qgrp(pi, vsi_node, qgrp_node, owner);
}

/**
 * ice_aq_add_lan_txq
 * @hw: pointer to the hardware structure
 * @num_qgrps: Number of added queue groups
 * @qg_list: list of queue groups to be added
 * @buf_size: size of buffer for indirect command
 * @cd: pointer to command details structure or NULL
 *
 * Add Tx LAN queue (0x0C30)
 *
 * NOTE:
 * Prior to calling add Tx LAN queue:
 * Initialize the following as part of the Tx queue context:
 * Completion queue ID if the queue uses Completion queue, Quanta profile,
 * Cache profile and Packet shaper profile.
 *
 * After add Tx LAN queue AQ command is completed:
 * Interrupts should be associated with specific queues,
 * Association of Tx queue to Doorbell queue is not part of Add LAN Tx queue
 * flow.
 */
enum ice_status
ice_aq_add_lan_txq(struct ice_hw *hw, uint8_t num_qgrps,
		   struct ice_aqc_add_tx_qgrp *qg_list, uint16_t buf_size,
		   struct ice_sq_cd *cd)
{
	struct ice_aqc_add_tx_qgrp *list;
	struct ice_aqc_add_txqs *cmd;
	struct ice_aq_desc desc;
	uint16_t i, sum_size = 0;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);

	cmd = &desc.params.add_txqs;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_add_txqs);

	if (!qg_list)
		return ICE_ERR_PARAM;

	if (num_qgrps > ICE_LAN_TXQ_MAX_QGRPS)
		return ICE_ERR_PARAM;

	for (i = 0, list = qg_list; i < num_qgrps; i++) {
		sum_size += ice_struct_size(list, txqs, list->num_txqs);
		list = (struct ice_aqc_add_tx_qgrp *)(list->txqs +
						      list->num_txqs);
	}

	if (buf_size != sum_size)
		return ICE_ERR_PARAM;

	desc.flags |= htole16(ICE_AQ_FLAG_RD);

	cmd->num_qgrps = num_qgrps;

	return ice_aq_send_cmd(hw, &desc, qg_list, buf_size, cd);
}

/**
 * ice_aq_add_sched_elems - adds scheduling element
 * @hw: pointer to the HW struct
 * @grps_req: the number of groups that are requested to be added
 * @buf: pointer to buffer
 * @buf_size: buffer size in bytes
 * @grps_added: returns total number of groups added
 * @cd: pointer to command details structure or NULL
 *
 * Add scheduling elements (0x0401)
 */
enum ice_status
ice_aq_add_sched_elems(struct ice_hw *hw, uint16_t grps_req,
		       struct ice_aqc_add_elem *buf, uint16_t buf_size,
		       uint16_t *grps_added, struct ice_sq_cd *cd)
{
	return ice_aqc_send_sched_elem_cmd(hw, ice_aqc_opc_add_sched_elems,
					   grps_req, (void *)buf, buf_size,
					   grps_added, cd);
}

/**
 * ice_aq_cfg_sched_elems - configures scheduler elements
 * @hw: pointer to the HW struct
 * @elems_req: number of elements to configure
 * @buf: pointer to buffer
 * @buf_size: buffer size in bytes
 * @elems_cfgd: returns total number of elements configured
 * @cd: pointer to command details structure or NULL
 *
 * Configure scheduling elements (0x0403)
 */
enum ice_status
ice_aq_cfg_sched_elems(struct ice_hw *hw, uint16_t elems_req,
		       struct ice_aqc_txsched_elem_data *buf, uint16_t buf_size,
		       uint16_t *elems_cfgd, struct ice_sq_cd *cd)
{
	return ice_aqc_send_sched_elem_cmd(hw, ice_aqc_opc_cfg_sched_elems,
					   elems_req, (void *)buf, buf_size,
					   elems_cfgd, cd);
}

/**
 * ice_sched_update_elem - update element
 * @hw: pointer to the HW struct
 * @node: pointer to node
 * @info: node info to update
 *
 * Update the HW DB, and local SW DB of node. Update the scheduling
 * parameters of node from argument info data buffer (Info->data buf) and
 * returns success or error on config sched element failure. The caller
 * needs to hold scheduler lock.
 */
enum ice_status
ice_sched_update_elem(struct ice_hw *hw, struct ice_sched_node *node,
		      struct ice_aqc_txsched_elem_data *info)
{
	struct ice_aqc_txsched_elem_data buf;
	enum ice_status status;
	uint16_t elem_cfgd = 0;
	uint16_t num_elems = 1;

	buf = *info;
	/* For TC nodes, CIR config is not supported */
	if (node->info.data.elem_type == ICE_AQC_ELEM_TYPE_TC)
		buf.data.valid_sections &= ~ICE_AQC_ELEM_VALID_CIR;
	/* Parent TEID is reserved field in this aq call */
	buf.parent_teid = 0;
	/* Element type is reserved field in this aq call */
	buf.data.elem_type = 0;
	/* Flags is reserved field in this aq call */
	buf.data.flags = 0;

	/* Update HW DB */
	/* Configure element node */
	status = ice_aq_cfg_sched_elems(hw, num_elems, &buf, sizeof(buf),
					&elem_cfgd, NULL);
	if (status || elem_cfgd != num_elems) {
		DNPRINTF(ICE_DBG_SCHED, "%s: Config sched elem error\n",
		    __func__);
		return ICE_ERR_CFG;
	}

	/* Config success case */
	/* Now update local SW DB */
	/* Only copy the data portion of info buffer */
	node->info.data = info->data;
	return status;
}

/**
 * ice_sched_replay_node_prio - re-configure node priority
 * @hw: pointer to the HW struct
 * @node: sched node to configure
 * @priority: priority value
 *
 * This function configures node element's priority value. It
 * needs to be called with scheduler lock held.
 */
enum ice_status
ice_sched_replay_node_prio(struct ice_hw *hw, struct ice_sched_node *node,
			   uint8_t priority)
{
	struct ice_aqc_txsched_elem_data buf;
	struct ice_aqc_txsched_elem *data;
	enum ice_status status;

	buf = node->info;
	data = &buf.data;
	data->valid_sections |= ICE_AQC_ELEM_VALID_GENERIC;
	data->generic = priority;

	/* Configure element */
	status = ice_sched_update_elem(hw, node, &buf);
	return status;
}

/**
 * ice_sched_rm_unused_rl_prof - remove unused RL profile
 * @hw: pointer to the hardware structure
 *
 * This function removes unused rate limit profiles from the HW and
 * SW DB. The caller needs to hold scheduler lock.
 */
void
ice_sched_rm_unused_rl_prof(struct ice_hw *hw)
{
	uint16_t ln;

	for (ln = 0; ln < hw->num_tx_sched_layers; ln++) {
		struct ice_aqc_rl_profile_info *rl_prof_elem;
		struct ice_aqc_rl_profile_info *rl_prof_tmp;

		TAILQ_FOREACH_SAFE(rl_prof_elem, &hw->rl_prof_list[ln],
		    list_entry, rl_prof_tmp) {
			if (!ice_sched_del_rl_profile(hw,
			    &hw->rl_prof_list[ln], rl_prof_elem))
				DNPRINTF(ICE_DBG_SCHED,
				    "%s: Removed rl profile\n", __func__);
		}
	}
}

/**
 * ice_sched_get_rl_prof_layer - selects rate limit profile creation layer
 * @pi: port information structure
 * @rl_type: type of rate limit BW - min, max, or shared
 * @layer_index: layer index
 *
 * This function returns requested profile creation layer.
 */
uint8_t
ice_sched_get_rl_prof_layer(struct ice_port_info *pi, enum ice_rl_type rl_type,
			    uint8_t layer_index)
{
	struct ice_hw *hw = pi->hw;

	if (layer_index >= hw->num_tx_sched_layers)
		return ICE_SCHED_INVAL_LAYER_NUM;
	switch (rl_type) {
	case ICE_MIN_BW:
		if (hw->layer_info[layer_index].max_cir_rl_profiles)
			return layer_index;
		break;
	case ICE_MAX_BW:
		if (hw->layer_info[layer_index].max_eir_rl_profiles)
			return layer_index;
		break;
	case ICE_SHARED_BW:
		/* if current layer doesn't support SRL profile creation
		 * then try a layer up or down.
		 */
		if (hw->layer_info[layer_index].max_srl_profiles)
			return layer_index;
		else if (layer_index < hw->num_tx_sched_layers - 1 &&
			 hw->layer_info[layer_index + 1].max_srl_profiles)
			return layer_index + 1;
		else if (layer_index > 0 &&
			 hw->layer_info[layer_index - 1].max_srl_profiles)
			return layer_index - 1;
		break;
	default:
		break;
	}
	return ICE_SCHED_INVAL_LAYER_NUM;
}

/**
 * ice_sched_get_node_rl_prof_id - get node's rate limit profile ID
 * @node: sched node
 * @rl_type: rate limit type
 *
 * If existing profile matches, it returns the corresponding rate
 * limit profile ID, otherwise it returns an invalid ID as error.
 */
uint16_t
ice_sched_get_node_rl_prof_id(struct ice_sched_node *node,
			      enum ice_rl_type rl_type)
{
	uint16_t rl_prof_id = ICE_SCHED_INVAL_PROF_ID;
	struct ice_aqc_txsched_elem *data;

	data = &node->info.data;
	switch (rl_type) {
	case ICE_MIN_BW:
		if (data->valid_sections & ICE_AQC_ELEM_VALID_CIR)
			rl_prof_id = le16toh(data->cir_bw.bw_profile_idx);
		break;
	case ICE_MAX_BW:
		if (data->valid_sections & ICE_AQC_ELEM_VALID_EIR)
			rl_prof_id = le16toh(data->eir_bw.bw_profile_idx);
		break;
	case ICE_SHARED_BW:
		if (data->valid_sections & ICE_AQC_ELEM_VALID_SHARED)
			rl_prof_id = le16toh(data->srl_id);
		break;
	default:
		break;
	}

	return rl_prof_id;
}

/**
 * ice_sched_cfg_node_bw_lmt - configure node sched params
 * @hw: pointer to the HW struct
 * @node: sched node to configure
 * @rl_type: rate limit type CIR, EIR, or shared
 * @rl_prof_id: rate limit profile ID
 *
 * This function configures node element's BW limit.
 */
enum ice_status
ice_sched_cfg_node_bw_lmt(struct ice_hw *hw, struct ice_sched_node *node,
			  enum ice_rl_type rl_type, uint16_t rl_prof_id)
{
	struct ice_aqc_txsched_elem_data buf;
	struct ice_aqc_txsched_elem *data;

	buf = node->info;
	data = &buf.data;
	switch (rl_type) {
	case ICE_MIN_BW:
		data->valid_sections |= ICE_AQC_ELEM_VALID_CIR;
		data->cir_bw.bw_profile_idx = htole16(rl_prof_id);
		break;
	case ICE_MAX_BW:
		data->valid_sections |= ICE_AQC_ELEM_VALID_EIR;
		data->eir_bw.bw_profile_idx = htole16(rl_prof_id);
		break;
	case ICE_SHARED_BW:
		data->valid_sections |= ICE_AQC_ELEM_VALID_SHARED;
		data->srl_id = htole16(rl_prof_id);
		break;
	default:
		/* Unknown rate limit type */
		return ICE_ERR_PARAM;
	}

	/* Configure element */
	return ice_sched_update_elem(hw, node, &buf);
}

/**
 * ice_sched_rm_rl_profile - remove RL profile ID
 * @hw: pointer to the hardware structure
 * @layer_num: layer number where profiles are saved
 * @profile_type: profile type like EIR, CIR, or SRL
 * @profile_id: profile ID to remove
 *
 * This function removes rate limit profile from layer 'layer_num' of type
 * 'profile_type' and profile ID as 'profile_id'. The caller needs to hold
 * scheduler lock.
 */
enum ice_status
ice_sched_rm_rl_profile(struct ice_hw *hw, uint8_t layer_num,
    uint8_t profile_type, uint16_t profile_id)
{
	struct ice_aqc_rl_profile_info *rl_prof_elem;
	enum ice_status status = ICE_SUCCESS;

	if (!hw || layer_num >= hw->num_tx_sched_layers)
		return ICE_ERR_PARAM;
	/* Check the existing list for RL profile */
	TAILQ_FOREACH(rl_prof_elem, &hw->rl_prof_list[layer_num], list_entry) {
		if ((rl_prof_elem->profile.flags & ICE_AQC_RL_PROFILE_TYPE_M) ==
		    profile_type &&
		    le16toh(rl_prof_elem->profile.profile_id) == profile_id) {
			if (rl_prof_elem->prof_id_ref)
				rl_prof_elem->prof_id_ref--;

			/* Remove old profile ID from database */
			status = ice_sched_del_rl_profile(hw,
			    &hw->rl_prof_list[layer_num], rl_prof_elem);
			if (status && status != ICE_ERR_IN_USE)
				DNPRINTF(ICE_DBG_SCHED,
				    "%s: Remove rl profile failed\n", __func__);
			break;
		}
	}
	if (status == ICE_ERR_IN_USE)
		status = ICE_SUCCESS;
	return status;
}

/**
 * ice_sched_set_node_bw_dflt - set node's bandwidth limit to default
 * @pi: port information structure
 * @node: pointer to node structure
 * @rl_type: rate limit type min, max, or shared
 * @layer_num: layer number where RL profiles are saved
 *
 * This function configures node element's BW rate limit profile ID of
 * type CIR, EIR, or SRL to default. This function needs to be called
 * with the scheduler lock held.
 */
enum ice_status
ice_sched_set_node_bw_dflt(struct ice_port_info *pi,
			   struct ice_sched_node *node,
			   enum ice_rl_type rl_type, uint8_t layer_num)
{
	enum ice_status status;
	struct ice_hw *hw;
	uint8_t profile_type;
	uint16_t rl_prof_id;
	uint16_t old_id;

	hw = pi->hw;
	switch (rl_type) {
	case ICE_MIN_BW:
		profile_type = ICE_AQC_RL_PROFILE_TYPE_CIR;
		rl_prof_id = ICE_SCHED_DFLT_RL_PROF_ID;
		break;
	case ICE_MAX_BW:
		profile_type = ICE_AQC_RL_PROFILE_TYPE_EIR;
		rl_prof_id = ICE_SCHED_DFLT_RL_PROF_ID;
		break;
	case ICE_SHARED_BW:
		profile_type = ICE_AQC_RL_PROFILE_TYPE_SRL;
		/* No SRL is configured for default case */
		rl_prof_id = ICE_SCHED_NO_SHARED_RL_PROF_ID;
		break;
	default:
		return ICE_ERR_PARAM;
	}
	/* Save existing RL prof ID for later clean up */
	old_id = ice_sched_get_node_rl_prof_id(node, rl_type);
	/* Configure BW scheduling parameters */
	status = ice_sched_cfg_node_bw_lmt(hw, node, rl_type, rl_prof_id);
	if (status)
		return status;

	/* Remove stale RL profile ID */
	if (old_id == ICE_SCHED_DFLT_RL_PROF_ID ||
	    old_id == ICE_SCHED_INVAL_PROF_ID)
		return ICE_SUCCESS;

	return ice_sched_rm_rl_profile(hw, layer_num, profile_type, old_id);
}

/**
 * ice_aq_add_rl_profile - adds rate limiting profile(s)
 * @hw: pointer to the HW struct
 * @num_profiles: the number of profile(s) to be add
 * @buf: pointer to buffer
 * @buf_size: buffer size in bytes
 * @num_profiles_added: total number of profiles added to return
 * @cd: pointer to command details structure
 *
 * Add RL profile (0x0410)
 */
enum ice_status
ice_aq_add_rl_profile(struct ice_hw *hw, uint16_t num_profiles,
		      struct ice_aqc_rl_profile_elem *buf, uint16_t buf_size,
		      uint16_t *num_profiles_added, struct ice_sq_cd *cd)
{
	return ice_aq_rl_profile(hw, ice_aqc_opc_add_rl_profiles, num_profiles,
				 buf, buf_size, num_profiles_added, cd);
}

/**
 * DIV_S64 - Divide signed 64-bit value with signed 64-bit divisor
 * @dividend: value to divide
 * @divisor: value to divide by
 *
 * Use DIV_S64 for any 64-bit divide which operates on signed 64-bit dividends.
 * Do not use this for unsigned 64-bit dividends as it will not produce
 * correct results if the dividend is larger than INT64_MAX.
 */
static inline int64_t DIV_S64(int64_t dividend, int64_t divisor)
{
	return dividend / divisor;
}

/**
 * DIV_U64 - Divide unsigned 64-bit value by unsigned 64-bit divisor
 * @dividend: value to divide
 * @divisor: value to divide by
 *
 * Use DIV_U64 for any 64-bit divide which operates on unsigned 64-bit
 * dividends. Do not use this for signed 64-bit dividends as it will not
 * handle negative values correctly.
 */
static inline uint64_t DIV_U64(uint64_t dividend, uint64_t divisor)
{
	return dividend / divisor;
}

static inline uint64_t round_up_64bit(uint64_t a, uint32_t b)
{
	return DIV_U64(((a) + (b) / 2), (b));
}

/**
 * ice_sched_calc_wakeup - calculate RL profile wakeup parameter
 * @hw: pointer to the HW struct
 * @bw: bandwidth in Kbps
 *
 * This function calculates the wakeup parameter of RL profile.
 */
uint16_t
ice_sched_calc_wakeup(struct ice_hw *hw, int32_t bw)
{
	int64_t bytes_per_sec, wakeup_int, wakeup_a, wakeup_b, wakeup_f;
	int32_t wakeup_f_int;
	uint16_t wakeup = 0;

	/* Get the wakeup integer value */
	bytes_per_sec = DIV_S64((int64_t)bw * 1000, 8);
	wakeup_int = DIV_S64(hw->psm_clk_freq, bytes_per_sec);
	if (wakeup_int > 63) {
		wakeup = (uint16_t)((1 << 15) | wakeup_int);
	} else {
		/* Calculate fraction value up to 4 decimals
		 * Convert Integer value to a constant multiplier
		 */
		wakeup_b = (int64_t)ICE_RL_PROF_MULTIPLIER * wakeup_int;
		wakeup_a = DIV_S64((int64_t)ICE_RL_PROF_MULTIPLIER *
				   hw->psm_clk_freq, bytes_per_sec);

		/* Get Fraction value */
		wakeup_f = wakeup_a - wakeup_b;

		/* Round up the Fractional value via Ceil(Fractional value) */
		if (wakeup_f > DIV_S64(ICE_RL_PROF_MULTIPLIER, 2))
			wakeup_f += 1;

		wakeup_f_int = (int32_t)DIV_S64(wakeup_f * ICE_RL_PROF_FRACTION,
					    ICE_RL_PROF_MULTIPLIER);
		wakeup |= (uint16_t)(wakeup_int << 9);
		wakeup |= (uint16_t)(0x1ff & wakeup_f_int);
	}

	return wakeup;
}

/**
 * ice_sched_bw_to_rl_profile - convert BW to profile parameters
 * @hw: pointer to the HW struct
 * @bw: bandwidth in Kbps
 * @profile: profile parameters to return
 *
 * This function converts the BW to profile structure format.
 */
enum ice_status
ice_sched_bw_to_rl_profile(struct ice_hw *hw, uint32_t bw,
			   struct ice_aqc_rl_profile_elem *profile)
{
	enum ice_status status = ICE_ERR_PARAM;
	int64_t bytes_per_sec, ts_rate, mv_tmp;
	bool found = false;
	int32_t encode = 0;
	int64_t mv = 0;
	int32_t i;

	/* Bw settings range is from 0.5Mb/sec to 100Gb/sec */
	if (bw < ICE_SCHED_MIN_BW || bw > ICE_SCHED_MAX_BW)
		return status;

	/* Bytes per second from Kbps */
	bytes_per_sec = DIV_S64((int64_t)bw * 1000, 8);

	/* encode is 6 bits but really useful are 5 bits */
	for (i = 0; i < 64; i++) {
		uint64_t pow_result = BIT_ULL(i);

		ts_rate = DIV_S64((int64_t)hw->psm_clk_freq,
				  pow_result * ICE_RL_PROF_TS_MULTIPLIER);
		if (ts_rate <= 0)
			continue;

		/* Multiplier value */
		mv_tmp = DIV_S64(bytes_per_sec * ICE_RL_PROF_MULTIPLIER,
				 ts_rate);

		/* Round to the nearest ICE_RL_PROF_MULTIPLIER */
		mv = round_up_64bit(mv_tmp, ICE_RL_PROF_MULTIPLIER);

		/* First multiplier value greater than the given
		 * accuracy bytes
		 */
		if (mv > ICE_RL_PROF_ACCURACY_BYTES) {
			encode = i;
			found = true;
			break;
		}
	}
	if (found) {
		uint16_t wm;

		wm = ice_sched_calc_wakeup(hw, bw);
		profile->rl_multiply = htole16(mv);
		profile->wake_up_calc = htole16(wm);
		profile->rl_encode = htole16(encode);
		status = ICE_SUCCESS;
	} else {
		status = ICE_ERR_DOES_NOT_EXIST;
	}

	return status;
}

/**
 * ice_sched_add_rl_profile - add RL profile
 * @hw: pointer to the hardware structure
 * @rl_type: type of rate limit BW - min, max, or shared
 * @bw: bandwidth in Kbps - Kilo bits per sec
 * @layer_num: specifies in which layer to create profile
 *
 * This function first checks the existing list for corresponding BW
 * parameter. If it exists, it returns the associated profile otherwise
 * it creates a new rate limit profile for requested BW, and adds it to
 * the HW DB and local list. It returns the new profile or null on error.
 * The caller needs to hold the scheduler lock.
 */
struct ice_aqc_rl_profile_info *
ice_sched_add_rl_profile(struct ice_hw *hw, enum ice_rl_type rl_type,
			 uint32_t bw, uint8_t layer_num)
{
	struct ice_aqc_rl_profile_info *rl_prof_elem;
	uint16_t profiles_added = 0, num_profiles = 1;
	struct ice_aqc_rl_profile_elem *buf;
	enum ice_status status;
	uint8_t profile_type;

	if (!hw || layer_num >= hw->num_tx_sched_layers)
		return NULL;
	switch (rl_type) {
	case ICE_MIN_BW:
		profile_type = ICE_AQC_RL_PROFILE_TYPE_CIR;
		break;
	case ICE_MAX_BW:
		profile_type = ICE_AQC_RL_PROFILE_TYPE_EIR;
		break;
	case ICE_SHARED_BW:
		profile_type = ICE_AQC_RL_PROFILE_TYPE_SRL;
		break;
	default:
		return NULL;
	}

	TAILQ_FOREACH(rl_prof_elem, &hw->rl_prof_list[layer_num], list_entry) {
		if ((rl_prof_elem->profile.flags & ICE_AQC_RL_PROFILE_TYPE_M) ==
		    profile_type && rl_prof_elem->bw == bw)
			/* Return existing profile ID info */
			return rl_prof_elem;
	}

	/* Create new profile ID */
	rl_prof_elem = (struct ice_aqc_rl_profile_info *)
		ice_malloc(hw, sizeof(*rl_prof_elem));
	if (!rl_prof_elem)
		return NULL;

	status = ice_sched_bw_to_rl_profile(hw, bw, &rl_prof_elem->profile);
	if (status != ICE_SUCCESS)
		goto exit_add_rl_prof;

	rl_prof_elem->bw = bw;
	/* layer_num is zero relative, and fw expects level from 1 to 9 */
	rl_prof_elem->profile.level = layer_num + 1;
	rl_prof_elem->profile.flags = profile_type;
	rl_prof_elem->profile.max_burst_size = htole16(hw->max_burst_size);

	/* Create new entry in HW DB */
	buf = &rl_prof_elem->profile;
	status = ice_aq_add_rl_profile(hw, num_profiles, buf, sizeof(*buf),
				       &profiles_added, NULL);
	if (status || profiles_added != num_profiles)
		goto exit_add_rl_prof;

	/* Good entry - add in the list */
	rl_prof_elem->prof_id_ref = 0;
	TAILQ_INSERT_HEAD(&hw->rl_prof_list[layer_num], rl_prof_elem,
	    list_entry);
	return rl_prof_elem;

exit_add_rl_prof:
	ice_free(hw, rl_prof_elem);
	return NULL;
}

/**
 * ice_sched_set_node_bw - set node's bandwidth
 * @pi: port information structure
 * @node: tree node
 * @rl_type: rate limit type min, max, or shared
 * @bw: bandwidth in Kbps - Kilo bits per sec
 * @layer_num: layer number
 *
 * This function adds new profile corresponding to requested BW, configures
 * node's RL profile ID of type CIR, EIR, or SRL, and removes old profile
 * ID from local database. The caller needs to hold scheduler lock.
 */
enum ice_status
ice_sched_set_node_bw(struct ice_port_info *pi, struct ice_sched_node *node,
		      enum ice_rl_type rl_type, uint32_t bw, uint8_t layer_num)
{
	struct ice_aqc_rl_profile_info *rl_prof_info;
	enum ice_status status = ICE_ERR_PARAM;
	struct ice_hw *hw = pi->hw;
	uint16_t old_id, rl_prof_id;

	rl_prof_info = ice_sched_add_rl_profile(hw, rl_type, bw, layer_num);
	if (!rl_prof_info)
		return status;

	rl_prof_id = le16toh(rl_prof_info->profile.profile_id);

	/* Save existing RL prof ID for later clean up */
	old_id = ice_sched_get_node_rl_prof_id(node, rl_type);
	/* Configure BW scheduling parameters */
	status = ice_sched_cfg_node_bw_lmt(hw, node, rl_type, rl_prof_id);
	if (status)
		return status;

	/* New changes has been applied */
	/* Increment the profile ID reference count */
	rl_prof_info->prof_id_ref++;

	/* Check for old ID removal */
	if ((old_id == ICE_SCHED_DFLT_RL_PROF_ID && rl_type != ICE_SHARED_BW) ||
	    old_id == ICE_SCHED_INVAL_PROF_ID || old_id == rl_prof_id)
		return ICE_SUCCESS;

	return ice_sched_rm_rl_profile(hw, layer_num,
				       rl_prof_info->profile.flags &
				       ICE_AQC_RL_PROFILE_TYPE_M, old_id);
}

/**
 * ice_sched_set_node_bw_lmt - set node's BW limit
 * @pi: port information structure
 * @node: tree node
 * @rl_type: rate limit type min, max, or shared
 * @bw: bandwidth in Kbps - Kilo bits per sec
 *
 * It updates node's BW limit parameters like BW RL profile ID of type CIR,
 * EIR, or SRL. The caller needs to hold scheduler lock.
 *
 * NOTE: Caller provides the correct SRL node in case of shared profile
 * settings.
 */
enum ice_status
ice_sched_set_node_bw_lmt(struct ice_port_info *pi, struct ice_sched_node *node,
			  enum ice_rl_type rl_type, uint32_t bw)
{
	struct ice_hw *hw;
	uint8_t layer_num;

	if (!pi)
		return ICE_ERR_PARAM;
	hw = pi->hw;
	/* Remove unused RL profile IDs from HW and SW DB */
	ice_sched_rm_unused_rl_prof(hw);

	layer_num = ice_sched_get_rl_prof_layer(pi, rl_type,
						node->tx_sched_layer);
	if (layer_num >= hw->num_tx_sched_layers)
		return ICE_ERR_PARAM;

	if (bw == ICE_SCHED_DFLT_BW)
		return ice_sched_set_node_bw_dflt(pi, node, rl_type, layer_num);

	return ice_sched_set_node_bw(pi, node, rl_type, bw, layer_num);
}

/**
 * ice_sched_cfg_node_bw_alloc - configure node BW weight/alloc params
 * @hw: pointer to the HW struct
 * @node: sched node to configure
 * @rl_type: rate limit type CIR, EIR, or shared
 * @bw_alloc: BW weight/allocation
 *
 * This function configures node element's BW allocation.
 */
enum ice_status
ice_sched_cfg_node_bw_alloc(struct ice_hw *hw, struct ice_sched_node *node,
			    enum ice_rl_type rl_type, uint16_t bw_alloc)
{
	struct ice_aqc_txsched_elem_data buf;
	struct ice_aqc_txsched_elem *data;
	enum ice_status status;

	buf = node->info;
	data = &buf.data;
	if (rl_type == ICE_MIN_BW) {
		data->valid_sections |= ICE_AQC_ELEM_VALID_CIR;
		data->cir_bw.bw_alloc = htole16(bw_alloc);
	} else if (rl_type == ICE_MAX_BW) {
		data->valid_sections |= ICE_AQC_ELEM_VALID_EIR;
		data->eir_bw.bw_alloc = htole16(bw_alloc);
	} else {
		return ICE_ERR_PARAM;
	}

	/* Configure element */
	status = ice_sched_update_elem(hw, node, &buf);
	return status;
}

/**
 * ice_sched_replay_node_bw - replay node(s) BW
 * @hw: pointer to the HW struct
 * @node: sched node to configure
 * @bw_t_info: BW type information
 *
 * This function restores node's BW from bw_t_info. The caller needs
 * to hold the scheduler lock.
 */
enum ice_status
ice_sched_replay_node_bw(struct ice_hw *hw, struct ice_sched_node *node,
			 struct ice_bw_type_info *bw_t_info)
{
	struct ice_port_info *pi = hw->port_info;
	enum ice_status status = ICE_ERR_PARAM;
	uint16_t bw_alloc;

	if (!node)
		return status;
	if (!ice_is_any_bit_set(bw_t_info->bw_t_bitmap, ICE_BW_TYPE_CNT))
		return ICE_SUCCESS;
	if (ice_is_bit_set(bw_t_info->bw_t_bitmap, ICE_BW_TYPE_PRIO)) {
		status = ice_sched_replay_node_prio(hw, node,
						    bw_t_info->generic);
		if (status)
			return status;
	}
	if (ice_is_bit_set(bw_t_info->bw_t_bitmap, ICE_BW_TYPE_CIR)) {
		status = ice_sched_set_node_bw_lmt(pi, node, ICE_MIN_BW,
						   bw_t_info->cir_bw.bw);
		if (status)
			return status;
	}
	if (ice_is_bit_set(bw_t_info->bw_t_bitmap, ICE_BW_TYPE_CIR_WT)) {
		bw_alloc = bw_t_info->cir_bw.bw_alloc;
		status = ice_sched_cfg_node_bw_alloc(hw, node, ICE_MIN_BW,
						     bw_alloc);
		if (status)
			return status;
	}
	if (ice_is_bit_set(bw_t_info->bw_t_bitmap, ICE_BW_TYPE_EIR)) {
		status = ice_sched_set_node_bw_lmt(pi, node, ICE_MAX_BW,
						   bw_t_info->eir_bw.bw);
		if (status)
			return status;
	}
	if (ice_is_bit_set(bw_t_info->bw_t_bitmap, ICE_BW_TYPE_EIR_WT)) {
		bw_alloc = bw_t_info->eir_bw.bw_alloc;
		status = ice_sched_cfg_node_bw_alloc(hw, node, ICE_MAX_BW,
						     bw_alloc);
		if (status)
			return status;
	}
	if (ice_is_bit_set(bw_t_info->bw_t_bitmap, ICE_BW_TYPE_SHARED))
		status = ice_sched_set_node_bw_lmt(pi, node, ICE_SHARED_BW,
						   bw_t_info->shared_bw);
	return status;
}
/**
 * ice_sched_replay_q_bw - replay queue type node BW
 * @pi: port information structure
 * @q_ctx: queue context structure
 *
 * This function replays queue type node bandwidth. This function needs to be
 * called with scheduler lock held.
 */
enum ice_status
ice_sched_replay_q_bw(struct ice_port_info *pi, struct ice_q_ctx *q_ctx)
{
	struct ice_sched_node *q_node;

	/* Following also checks the presence of node in tree */
	q_node = ice_sched_find_node_by_teid(pi->root, q_ctx->q_teid);
	if (!q_node)
		return ICE_ERR_PARAM;
	return ice_sched_replay_node_bw(pi->hw, q_node, &q_ctx->bw_t_info);
}

/**
 * ice_ena_vsi_txq
 * @pi: port information structure
 * @vsi_handle: software VSI handle
 * @tc: TC number
 * @q_handle: software queue handle
 * @num_qgrps: Number of added queue groups
 * @buf: list of queue groups to be added
 * @buf_size: size of buffer for indirect command
 * @cd: pointer to command details structure or NULL
 *
 * This function adds one LAN queue
 */
enum ice_status
ice_ena_vsi_txq(struct ice_port_info *pi, uint16_t vsi_handle, uint8_t tc,
    uint16_t q_handle, uint8_t num_qgrps, struct ice_aqc_add_tx_qgrp *buf,
    uint16_t buf_size, struct ice_sq_cd *cd)
{
	struct ice_aqc_txsched_elem_data node = { 0 };
	struct ice_sched_node *parent;
	struct ice_q_ctx *q_ctx;
	enum ice_status status;
	struct ice_hw *hw;

	if (!pi || pi->port_state != ICE_SCHED_PORT_STATE_READY)
		return ICE_ERR_CFG;

	if (num_qgrps > 1 || buf->num_txqs > 1)
		return ICE_ERR_MAX_LIMIT;

	hw = pi->hw;

	if (!ice_is_vsi_valid(hw, vsi_handle))
		return ICE_ERR_PARAM;
#if 0
	ice_acquire_lock(&pi->sched_lock);
#endif
	q_ctx = ice_get_lan_q_ctx(hw, vsi_handle, tc, q_handle);
	if (!q_ctx) {
		DNPRINTF(ICE_DBG_SCHED, "%s: Enaq: invalid queue handle %d\n",
		    __func__, q_handle);
		status = ICE_ERR_PARAM;
		goto ena_txq_exit;
	}

	/* find a parent node */
	parent = ice_sched_get_free_qparent(pi, vsi_handle, tc,
					    ICE_SCHED_NODE_OWNER_LAN);
	if (!parent) {
		status = ICE_ERR_PARAM;
		goto ena_txq_exit;
	}

	buf->parent_teid = parent->info.node_teid;
	node.parent_teid = parent->info.node_teid;
	/* Mark that the values in the "generic" section as valid. The default
	 * value in the "generic" section is zero. This means that :
	 * - Scheduling mode is Bytes Per Second (BPS), indicated by Bit 0.
	 * - 0 priority among siblings, indicated by Bit 1-3.
	 * - WFQ, indicated by Bit 4.
	 * - 0 Adjustment value is used in PSM credit update flow, indicated by
	 * Bit 5-6.
	 * - Bit 7 is reserved.
	 * Without setting the generic section as valid in valid_sections, the
	 * Admin queue command will fail with error code ICE_AQ_RC_EINVAL.
	 */
	buf->txqs[0].info.valid_sections =
		ICE_AQC_ELEM_VALID_GENERIC | ICE_AQC_ELEM_VALID_CIR |
		ICE_AQC_ELEM_VALID_EIR;
	buf->txqs[0].info.generic = 0;
	buf->txqs[0].info.cir_bw.bw_profile_idx =
		htole16(ICE_SCHED_DFLT_RL_PROF_ID);
	buf->txqs[0].info.cir_bw.bw_alloc =
		htole16(ICE_SCHED_DFLT_BW_WT);
	buf->txqs[0].info.eir_bw.bw_profile_idx =
		htole16(ICE_SCHED_DFLT_RL_PROF_ID);
	buf->txqs[0].info.eir_bw.bw_alloc =
		htole16(ICE_SCHED_DFLT_BW_WT);

	/* add the LAN queue */
	status = ice_aq_add_lan_txq(hw, num_qgrps, buf, buf_size, cd);
	if (status != ICE_SUCCESS) {
		DNPRINTF(ICE_DBG_SCHED, "%s: enable queue %d failed %d\n",
		    __func__, le16toh(buf->txqs[0].txq_id),
		    hw->adminq.sq_last_status);
		goto ena_txq_exit;
	}

	node.node_teid = buf->txqs[0].q_teid;
	node.data.elem_type = ICE_AQC_ELEM_TYPE_LEAF;
	q_ctx->q_handle = q_handle;
	q_ctx->q_teid = le32toh(node.node_teid);

	/* add a leaf node into scheduler tree queue layer */
	status = ice_sched_add_node(pi, hw->num_tx_sched_layers - 1, &node,
	    NULL);
	if (!status)
		status = ice_sched_replay_q_bw(pi, q_ctx);

ena_txq_exit:
#if 0
	ice_release_lock(&pi->sched_lock);
#endif
	return status;
}

/* LAN Tx Queue Context used for set Tx config by ice_aqc_opc_add_txqs,
 * Bit[0-175] is valid
 */
const struct ice_ctx_ele ice_tlan_ctx_info[] = {
				    /* Field			Width	LSB */
	ICE_CTX_STORE(ice_tlan_ctx, base,			57,	0),
	ICE_CTX_STORE(ice_tlan_ctx, port_num,			3,	57),
	ICE_CTX_STORE(ice_tlan_ctx, cgd_num,			5,	60),
	ICE_CTX_STORE(ice_tlan_ctx, pf_num,			3,	65),
	ICE_CTX_STORE(ice_tlan_ctx, vmvf_num,			10,	68),
	ICE_CTX_STORE(ice_tlan_ctx, vmvf_type,			2,	78),
	ICE_CTX_STORE(ice_tlan_ctx, src_vsi,			10,	80),
	ICE_CTX_STORE(ice_tlan_ctx, tsyn_ena,			1,	90),
	ICE_CTX_STORE(ice_tlan_ctx, internal_usage_flag,	1,	91),
	ICE_CTX_STORE(ice_tlan_ctx, alt_vlan,			1,	92),
	ICE_CTX_STORE(ice_tlan_ctx, cpuid,			8,	93),
	ICE_CTX_STORE(ice_tlan_ctx, wb_mode,			1,	101),
	ICE_CTX_STORE(ice_tlan_ctx, tphrd_desc,			1,	102),
	ICE_CTX_STORE(ice_tlan_ctx, tphrd,			1,	103),
	ICE_CTX_STORE(ice_tlan_ctx, tphwr_desc,			1,	104),
	ICE_CTX_STORE(ice_tlan_ctx, cmpq_id,			9,	105),
	ICE_CTX_STORE(ice_tlan_ctx, qnum_in_func,		14,	114),
	ICE_CTX_STORE(ice_tlan_ctx, itr_notification_mode,	1,	128),
	ICE_CTX_STORE(ice_tlan_ctx, adjust_prof_id,		6,	129),
	ICE_CTX_STORE(ice_tlan_ctx, qlen,			13,	135),
	ICE_CTX_STORE(ice_tlan_ctx, quanta_prof_idx,		4,	148),
	ICE_CTX_STORE(ice_tlan_ctx, tso_ena,			1,	152),
	ICE_CTX_STORE(ice_tlan_ctx, tso_qnum,			11,	153),
	ICE_CTX_STORE(ice_tlan_ctx, legacy_int,			1,	164),
	ICE_CTX_STORE(ice_tlan_ctx, drop_ena,			1,	165),
	ICE_CTX_STORE(ice_tlan_ctx, cache_prof_idx,		2,	166),
	ICE_CTX_STORE(ice_tlan_ctx, pkt_shaper_prof_idx,	3,	168),
	ICE_CTX_STORE(ice_tlan_ctx, int_q_state,		122,	171),
	{ 0 }
};

/**
 * ice_cfg_vsi_for_tx - Configure the hardware for Tx
 * @vsi: the VSI to configure
 *
 * Configure the device Tx queues through firmware AdminQ commands. After
 * this, Tx queues will be ready for transmit.
 */
int
ice_cfg_vsi_for_tx(struct ice_vsi *vsi)
{
	struct ice_softc *sc = vsi->sc;
	struct ice_aqc_add_tx_qgrp *qg;
	struct ice_hw *hw = &vsi->sc->hw;
	enum ice_status status;
	int i;
	int err = 0;
	uint16_t qg_size, pf_q;

	qg_size = ice_struct_size(qg, txqs, 1);
	qg = (struct ice_aqc_add_tx_qgrp *)malloc(qg_size, M_DEVBUF,
	    M_NOWAIT|M_ZERO);
	if (!qg)
		return (ENOMEM);

	qg->num_txqs = 1;

	for (i = 0; i < vsi->num_tx_queues; i++) {
		struct ice_tlan_ctx tlan_ctx = { 0 };
		struct ice_tx_queue *txq = &vsi->tx_queues[i];

		pf_q = vsi->tx_qmap[txq->me];
		qg->txqs[0].txq_id = htole16(pf_q);

		err = ice_setup_tx_ctx(txq, &tlan_ctx, pf_q);
		if (err)
			goto free_txqg;

		ice_set_ctx(hw, (uint8_t *)&tlan_ctx, qg->txqs[0].txq_ctx,
			    ice_tlan_ctx_info);

		status = ice_ena_vsi_txq(hw->port_info, vsi->idx, txq->tc,
					 txq->q_handle, 1, qg, qg_size, NULL);
		if (status) {
			printf("%s: Failed to set LAN Tx queue %d "
			    "(TC %d, handle %d) context, err %s aq_err %s\n",
			    sc->sc_dev.dv_xname, i, txq->tc, txq->q_handle,
			    ice_status_str(status),
			    ice_aq_str(hw->adminq.sq_last_status));
			err = ENODEV;
			goto free_txqg;
		}

		/* Keep track of the Tx queue TEID */
		if (pf_q == le16toh(qg->txqs[0].txq_id))
			txq->q_teid = le32toh(qg->txqs[0].q_teid);
	}

free_txqg:
	free(qg, M_DEVBUF, qg_size);

	return (err);
}

/* Different reset sources for which a disable queue AQ call has to be made in
 * order to clean the Tx scheduler as a part of the reset
 */
enum ice_disq_rst_src {
	ICE_NO_RESET = 0,
	ICE_VM_RESET,
	ICE_VF_RESET,
};

/**
 * ice_aq_dis_lan_txq
 * @hw: pointer to the hardware structure
 * @num_qgrps: number of groups in the list
 * @qg_list: the list of groups to disable
 * @buf_size: the total size of the qg_list buffer in bytes
 * @rst_src: if called due to reset, specifies the reset source
 * @vmvf_num: the relative VM or VF number that is undergoing the reset
 * @cd: pointer to command details structure or NULL
 *
 * Disable LAN Tx queue (0x0C31)
 */
enum ice_status
ice_aq_dis_lan_txq(struct ice_hw *hw, uint8_t num_qgrps,
		   struct ice_aqc_dis_txq_item *qg_list, uint16_t buf_size,
		   enum ice_disq_rst_src rst_src, uint16_t vmvf_num,
		   struct ice_sq_cd *cd)
{
	struct ice_aqc_dis_txq_item *item;
	struct ice_aqc_dis_txqs *cmd;
	struct ice_aq_desc desc;
	enum ice_status status;
	uint16_t i, sz = 0;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);
	cmd = &desc.params.dis_txqs;
	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_dis_txqs);

	/* qg_list can be NULL only in VM/VF reset flow */
	if (!qg_list && !rst_src)
		return ICE_ERR_PARAM;

	if (num_qgrps > ICE_LAN_TXQ_MAX_QGRPS)
		return ICE_ERR_PARAM;

	cmd->num_entries = num_qgrps;

	cmd->vmvf_and_timeout = htole16((5 << ICE_AQC_Q_DIS_TIMEOUT_S) &
	    ICE_AQC_Q_DIS_TIMEOUT_M);

	switch (rst_src) {
	case ICE_VM_RESET:
		cmd->cmd_type = ICE_AQC_Q_DIS_CMD_VM_RESET;
		cmd->vmvf_and_timeout |=
			htole16(vmvf_num & ICE_AQC_Q_DIS_VMVF_NUM_M);
		break;
	case ICE_VF_RESET:
		cmd->cmd_type = ICE_AQC_Q_DIS_CMD_VF_RESET;
		/* In this case, FW expects vmvf_num to be absolute VF ID */
		cmd->vmvf_and_timeout |=
			htole16((vmvf_num + hw->func_caps.vf_base_id) &
			    ICE_AQC_Q_DIS_VMVF_NUM_M);
		break;
	case ICE_NO_RESET:
	default:
		break;
	}

	/* flush pipe on time out */
	cmd->cmd_type |= ICE_AQC_Q_DIS_CMD_FLUSH_PIPE;
	/* If no queue group info, we are in a reset flow. Issue the AQ */
	if (!qg_list)
		goto do_aq;

	/* set RD bit to indicate that command buffer is provided by the driver
	 * and it needs to be read by the firmware
	 */
	desc.flags |= htole16(ICE_AQ_FLAG_RD);

	for (i = 0, item = qg_list; i < num_qgrps; i++) {
		uint16_t item_size = ice_struct_size(item, q_id, item->num_qs);

		/* If the num of queues is even, add 2 bytes of padding */
		if ((item->num_qs % 2) == 0)
			item_size += 2;

		sz += item_size;

		item = (struct ice_aqc_dis_txq_item *)((uint8_t *)item +
		    item_size);
	}

	if (buf_size != sz)
		return ICE_ERR_PARAM;

do_aq:
	status = ice_aq_send_cmd(hw, &desc, qg_list, buf_size, cd);
	if (status) {
		if (!qg_list)
			DNPRINTF(ICE_DBG_SCHED, "%s: VM%d disable failed %d\n", 
			    __func__, vmvf_num, hw->adminq.sq_last_status);
		else
			DNPRINTF(ICE_DBG_SCHED,
			    "%s: disable queue %d failed %d\n", __func__,
			    le16toh(qg_list[0].q_id[0]),
			    hw->adminq.sq_last_status);
	}

	return status;
}

/**
 * ice_dis_vsi_txq
 * @pi: port information structure
 * @vsi_handle: software VSI handle
 * @tc: TC number
 * @num_queues: number of queues
 * @q_handles: pointer to software queue handle array
 * @q_ids: pointer to the q_id array
 * @q_teids: pointer to queue node teids
 * @rst_src: if called due to reset, specifies the reset source
 * @vmvf_num: the relative VM or VF number that is undergoing the reset
 * @cd: pointer to command details structure or NULL
 *
 * This function removes queues and their corresponding nodes in SW DB
 */
enum ice_status
ice_dis_vsi_txq(struct ice_port_info *pi, uint16_t vsi_handle, uint8_t tc,
    uint8_t num_queues, uint16_t *q_handles, uint16_t *q_ids,
    uint32_t *q_teids, enum ice_disq_rst_src rst_src, uint16_t vmvf_num,
    struct ice_sq_cd *cd)
{
	enum ice_status status = ICE_ERR_DOES_NOT_EXIST;
	struct ice_aqc_dis_txq_item *qg_list;
	struct ice_q_ctx *q_ctx;
	struct ice_hw *hw;
	uint16_t i, buf_size;

	if (!pi || pi->port_state != ICE_SCHED_PORT_STATE_READY)
		return ICE_ERR_CFG;

	hw = pi->hw;

	if (!num_queues) {
		/* if queue is disabled already yet the disable queue command
		 * has to be sent to complete the VF reset, then call
		 * ice_aq_dis_lan_txq without any queue information
		 */
		if (rst_src)
			return ice_aq_dis_lan_txq(hw, 0, NULL, 0, rst_src,
						  vmvf_num, NULL);
		return ICE_ERR_CFG;
	}

	buf_size = ice_struct_size(qg_list, q_id, 1);
	qg_list = (struct ice_aqc_dis_txq_item *)ice_malloc(hw, buf_size);
	if (!qg_list)
		return ICE_ERR_NO_MEMORY;
#if 0
	ice_acquire_lock(&pi->sched_lock);
#endif
	for (i = 0; i < num_queues; i++) {
		struct ice_sched_node *node;

		node = ice_sched_find_node_by_teid(pi->root, q_teids[i]);
		if (!node)
			continue;
		q_ctx = ice_get_lan_q_ctx(hw, vsi_handle, tc, q_handles[i]);
		if (!q_ctx) {
			DNPRINTF(ICE_DBG_SCHED, "%s: invalid queue handle%d\n",
			    __func__, q_handles[i]);
			continue;
		}
		if (q_ctx->q_handle != q_handles[i]) {
			DNPRINTF(ICE_DBG_SCHED, "%s: Err:handles %d %d\n",
			    __func__, q_ctx->q_handle, q_handles[i]);
			continue;
		}
		qg_list->parent_teid = node->info.parent_teid;
		qg_list->num_qs = 1;
		qg_list->q_id[0] = htole16(q_ids[i]);
		status = ice_aq_dis_lan_txq(hw, 1, qg_list, buf_size, rst_src,
					    vmvf_num, cd);

		if (status != ICE_SUCCESS)
			break;
		ice_free_sched_node(pi, node);
		q_ctx->q_handle = ICE_INVAL_Q_HANDLE;
	}
#if 0
	ice_release_lock(&pi->sched_lock);
#endif
	ice_free(hw, qg_list);
	return status;
}

/**
 * ice_vsi_disable_tx - Disable (unconfigure) Tx queues for a VSI
 * @vsi: the VSI to disable
 *
 * Disables the Tx queues associated with this VSI. Essentially the opposite
 * of ice_cfg_vsi_for_tx.
 */
int
ice_vsi_disable_tx(struct ice_vsi *vsi)
{
	struct ice_softc *sc = vsi->sc;
	struct ice_hw *hw = &sc->hw;
	enum ice_status status;
	uint32_t *q_teids;
	uint16_t *q_ids, *q_handles;
	size_t q_teids_size, q_ids_size, q_handles_size;
	int tc, j, buf_idx, err = 0;

	if (vsi->num_tx_queues > 255)
		return (ENOSYS);

	q_teids_size = sizeof(*q_teids) * vsi->num_tx_queues;
	q_teids = (uint32_t *)malloc(q_teids_size, M_DEVBUF, M_NOWAIT|M_ZERO);
	if (!q_teids)
		return (ENOMEM);

	q_ids_size = sizeof(*q_ids) * vsi->num_tx_queues;
	q_ids = (uint16_t *)malloc(q_ids_size, M_DEVBUF, M_NOWAIT|M_ZERO);
	if (!q_ids) {
		err = (ENOMEM);
		goto free_q_teids;
	}

	q_handles_size = sizeof(*q_handles) * vsi->num_tx_queues;
	q_handles = (uint16_t *)malloc(q_handles_size, M_DEVBUF,
	    M_NOWAIT|M_ZERO);
	if (!q_handles) {
		err = (ENOMEM);
		goto free_q_ids;
	}

	ice_for_each_traffic_class(tc) {
		struct ice_tc_info *tc_info = &vsi->tc_info[tc];
		uint16_t start_idx, end_idx;

		/* Skip rest of disabled TCs once the first
		 * disabled TC is found */
		if (!(vsi->tc_map & BIT(tc)))
			break;

		/* Fill out TX queue information for this TC */
		start_idx = tc_info->qoffset;
		end_idx = start_idx + tc_info->qcount_tx;
		buf_idx = 0;
		for (j = start_idx; j < end_idx; j++) {
			struct ice_tx_queue *txq = &vsi->tx_queues[j];

			q_ids[buf_idx] = vsi->tx_qmap[j];
			q_handles[buf_idx] = txq->q_handle;
			q_teids[buf_idx] = txq->q_teid;
			buf_idx++;
		}

		status = ice_dis_vsi_txq(hw->port_info, vsi->idx, tc, buf_idx,
					 q_handles, q_ids, q_teids,
					 ICE_NO_RESET, 0, NULL);
		if (status == ICE_ERR_DOES_NOT_EXIST) {
			; /* Queues have been disabled */
		} else if (status == ICE_ERR_RESET_ONGOING) {
			DPRINTF("%s: Reset in progress. LAN Tx queues already "
			    "disabled\n", __func__);
			break;
		} else if (status) {
			printf("%s: Failed to disable LAN Tx queues: "
			    "err %s aq_err %s\n", sc->sc_dev.dv_xname,
			    ice_status_str(status),
			    ice_aq_str(hw->adminq.sq_last_status));
			err = (ENODEV);
			break;
		}

		/* Clear buffers */
		memset(q_teids, 0, q_teids_size);
		memset(q_ids, 0, q_ids_size);
		memset(q_handles, 0, q_handles_size);
	}

/* free_q_handles: */
	free(q_handles, M_DEVBUF, q_handles_size);
free_q_ids:
	free(q_ids, M_DEVBUF, q_ids_size);
free_q_teids:
	free(q_teids, M_DEVBUF, q_teids_size);

	return err;
}

/**
 * ice_copy_rxq_ctx_to_hw
 * @hw: pointer to the hardware structure
 * @ice_rxq_ctx: pointer to the rxq context
 * @rxq_index: the index of the Rx queue
 *
 * Copies rxq context from dense structure to HW register space
 */
enum ice_status
ice_copy_rxq_ctx_to_hw(struct ice_hw *hw, uint8_t *ice_rxq_ctx,
    uint32_t rxq_index)
{
	uint8_t i;

	if (!ice_rxq_ctx)
		return ICE_ERR_BAD_PTR;

	if (rxq_index > QRX_CTRL_MAX_INDEX)
		return ICE_ERR_PARAM;

	/* Copy each dword separately to HW */
	for (i = 0; i < ICE_RXQ_CTX_SIZE_DWORDS; i++) {
		ICE_WRITE(hw, QRX_CONTEXT(i, rxq_index),
		     *((uint32_t *)(ice_rxq_ctx + (i * sizeof(uint32_t)))));

		DNPRINTF(ICE_DBG_QCTX, "%s: qrxdata[%d]: %08X\n", __func__,
		    i, *((uint32_t *)(ice_rxq_ctx + (i * sizeof(uint32_t)))));
	}

	return ICE_SUCCESS;
}

/* LAN Rx Queue Context */
static const struct ice_ctx_ele ice_rlan_ctx_info[] = {
	/* Field		Width	LSB */
	ICE_CTX_STORE(ice_rlan_ctx, head,		13,	0),
	ICE_CTX_STORE(ice_rlan_ctx, cpuid,		8,	13),
	ICE_CTX_STORE(ice_rlan_ctx, base,		57,	32),
	ICE_CTX_STORE(ice_rlan_ctx, qlen,		13,	89),
	ICE_CTX_STORE(ice_rlan_ctx, dbuf,		7,	102),
	ICE_CTX_STORE(ice_rlan_ctx, hbuf,		5,	109),
	ICE_CTX_STORE(ice_rlan_ctx, dtype,		2,	114),
	ICE_CTX_STORE(ice_rlan_ctx, dsize,		1,	116),
	ICE_CTX_STORE(ice_rlan_ctx, crcstrip,		1,	117),
	ICE_CTX_STORE(ice_rlan_ctx, l2tsel,		1,	119),
	ICE_CTX_STORE(ice_rlan_ctx, hsplit_0,		4,	120),
	ICE_CTX_STORE(ice_rlan_ctx, hsplit_1,		2,	124),
	ICE_CTX_STORE(ice_rlan_ctx, showiv,		1,	127),
	ICE_CTX_STORE(ice_rlan_ctx, rxmax,		14,	174),
	ICE_CTX_STORE(ice_rlan_ctx, tphrdesc_ena,	1,	193),
	ICE_CTX_STORE(ice_rlan_ctx, tphwdesc_ena,	1,	194),
	ICE_CTX_STORE(ice_rlan_ctx, tphdata_ena,	1,	195),
	ICE_CTX_STORE(ice_rlan_ctx, tphhead_ena,	1,	196),
	ICE_CTX_STORE(ice_rlan_ctx, lrxqthresh,		3,	198),
	ICE_CTX_STORE(ice_rlan_ctx, prefena,		1,	201),
	{ 0 }
};

/**
 * ice_write_rxq_ctx
 * @hw: pointer to the hardware structure
 * @rlan_ctx: pointer to the rxq context
 * @rxq_index: the index of the Rx queue
 *
 * Converts rxq context from sparse to dense structure and then writes
 * it to HW register space and enables the hardware to prefetch descriptors
 * instead of only fetching them on demand
 */
enum ice_status
ice_write_rxq_ctx(struct ice_hw *hw, struct ice_rlan_ctx *rlan_ctx,
		  uint32_t rxq_index)
{
	uint8_t ctx_buf[ICE_RXQ_CTX_SZ] = { 0 };

	if (!rlan_ctx)
		return ICE_ERR_BAD_PTR;

	rlan_ctx->prefena = 1;

	ice_set_ctx(hw, (uint8_t *)rlan_ctx, ctx_buf, ice_rlan_ctx_info);
	return ice_copy_rxq_ctx_to_hw(hw, ctx_buf, rxq_index);
}

/**
 * ice_setup_rx_ctx - Setup an Rx context structure for a receive queue
 * @rxq: the receive queue to program
 *
 * Setup an Rx queue context structure and program it into the hardware
 * registers. This is a necessary step for enabling the Rx queue.
 */
int
ice_setup_rx_ctx(struct ice_rx_queue *rxq)
{
	struct ice_rlan_ctx rlan_ctx = {0};
	struct ice_vsi *vsi = rxq->vsi;
	struct ice_softc *sc = vsi->sc;
	struct ice_hw *hw = &sc->hw;
	enum ice_status status;
	uint32_t rxdid = ICE_RXDID_FLEX_NIC;
	uint32_t regval;
	uint16_t pf_q;

	pf_q = vsi->rx_qmap[rxq->me];

	/* set the receive queue base address, defined in 128 byte units */
	rlan_ctx.base = rxq->rx_paddr >> 7;

	rlan_ctx.qlen = rxq->desc_count;

	rlan_ctx.dbuf = vsi->mbuf_sz >> ICE_RLAN_CTX_DBUF_S;

	/* use 32 byte descriptors */
	rlan_ctx.dsize = 1;

	/* Strip the Ethernet CRC bytes before the packet is posted to the
	 * host memory.
	 */
	rlan_ctx.crcstrip = 1;

	rlan_ctx.l2tsel = 1;

	/* don't do header splitting */
	rlan_ctx.dtype = ICE_RX_DTYPE_NO_SPLIT;
	rlan_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_NO_SPLIT;
	rlan_ctx.hsplit_1 = ICE_RLAN_RX_HSPLIT_1_NO_SPLIT;

	/* strip VLAN from inner headers */
	rlan_ctx.showiv = 1;

	rlan_ctx.rxmax = MIN(vsi->max_frame_size,
			     ICE_MAX_RX_SEGS * vsi->mbuf_sz);

	rlan_ctx.lrxqthresh = 1;

	if (vsi->type != ICE_VSI_VF) {
		regval = ICE_READ(hw, QRXFLXP_CNTXT(pf_q));
		regval &= ~QRXFLXP_CNTXT_RXDID_IDX_M;
		regval |= (rxdid << QRXFLXP_CNTXT_RXDID_IDX_S) &
			QRXFLXP_CNTXT_RXDID_IDX_M;

		regval &= ~QRXFLXP_CNTXT_RXDID_PRIO_M;
		regval |= (0x03 << QRXFLXP_CNTXT_RXDID_PRIO_S) &
			QRXFLXP_CNTXT_RXDID_PRIO_M;

		ICE_WRITE(hw, QRXFLXP_CNTXT(pf_q), regval);
	}

	status = ice_write_rxq_ctx(hw, &rlan_ctx, pf_q);
	if (status) {
		printf("%s: Failed to set LAN Rx queue context, "
		    "err %s aq_err %s\n", sc->sc_dev.dv_xname,
		    ice_status_str(status),
		    ice_aq_str(hw->adminq.sq_last_status));
		return (EIO);
	}

	ICE_WRITE(hw, rxq->tail, 0);

	return 0;
}

/**
 * ice_cfg_vsi_for_rx - Configure the hardware for Rx
 * @vsi: the VSI to configure
 *
 * Prepare an Rx context descriptor and configure the device to receive
 * traffic.
 */
int
ice_cfg_vsi_for_rx(struct ice_vsi *vsi)
{
	int i, err;

	for (i = 0; i < vsi->num_rx_queues; i++) {
		err = ice_setup_rx_ctx(&vsi->rx_queues[i]);
		if (err)
			return err;
	}

	return (0);
}

/**
 * ice_is_rxq_ready - Check if an Rx queue is ready
 * @hw: ice hw structure
 * @pf_q: absolute PF queue index to check
 * @reg: on successful return, contains qrx_ctrl contents
 *
 * Reads the QRX_CTRL register and verifies if the queue is in a consistent
 * state. That is, QENA_REQ matches QENA_STAT. Used to check before making
 * a request to change the queue, as well as to verify the request has
 * finished. The queue should change status within a few microseconds, so we
 * use a small delay while polling the register.
 *
 * Returns an error code if the queue does not update after a few retries.
 */
int
ice_is_rxq_ready(struct ice_hw *hw, int pf_q, uint32_t *reg)
{
	uint32_t qrx_ctrl, qena_req, qena_stat;
	int i;

	for (i = 0; i < ICE_Q_WAIT_RETRY_LIMIT; i++) {
		qrx_ctrl = ICE_READ(hw, QRX_CTRL(pf_q));
		qena_req = (qrx_ctrl >> QRX_CTRL_QENA_REQ_S) & 1;
		qena_stat = (qrx_ctrl >> QRX_CTRL_QENA_STAT_S) & 1;

		/* if the request and status bits equal, then the queue is
		 * fully disabled or enabled.
		 */
		if (qena_req == qena_stat) {
			*reg = qrx_ctrl;
			return (0);
		}

		/* wait a few microseconds before we check again */
		DELAY(10);
	}

	return (ETIMEDOUT);
}

/**
 * ice_control_rx_queue - Configure hardware to start or stop an Rx queue
 * @vsi: VSI containing queue to enable/disable
 * @qidx: Queue index in VSI space
 * @enable: true to enable queue, false to disable
 *
 * Control the Rx queue through the QRX_CTRL register, enabling or disabling
 * it. Wait for the appropriate time to ensure that the queue has actually
 * reached the expected state.
 */
int
ice_control_rx_queue(struct ice_vsi *vsi, uint16_t qidx, bool enable)
{
	struct ice_softc *sc = vsi->sc;
	struct ice_hw *hw = &sc->hw;
	uint32_t qrx_ctrl = 0;
	int err;
	struct ice_rx_queue *rxq = &vsi->rx_queues[qidx];
	int pf_q = vsi->rx_qmap[rxq->me];

	err = ice_is_rxq_ready(hw, pf_q, &qrx_ctrl);
	if (err) {
		printf("%s: Rx queue %d is not ready\n",
		    sc->sc_dev.dv_xname, pf_q);
		return err;
	}

	/* Skip if the queue is already in correct state */
	if (enable == !!(qrx_ctrl & QRX_CTRL_QENA_STAT_M))
		return (0);

	if (enable)
		qrx_ctrl |= QRX_CTRL_QENA_REQ_M;
	else
		qrx_ctrl &= ~QRX_CTRL_QENA_REQ_M;
	ICE_WRITE(hw, QRX_CTRL(pf_q), qrx_ctrl);

	/* wait for the queue to finalize the request */
	err = ice_is_rxq_ready(hw, pf_q, &qrx_ctrl);
	if (err) {
		printf("%s: Rx queue %d %sable timeout\n",
		    sc->sc_dev.dv_xname, pf_q, (enable ? "en" : "dis"));
		return err;
	}

	/* this should never happen */
	if (enable != !!(qrx_ctrl & QRX_CTRL_QENA_STAT_M)) {
		printf("%s: Rx queue %d invalid state\n",
		    sc->sc_dev.dv_xname, pf_q);
		return (EINVAL);
	}

	return (0);
}

/**
 * ice_control_all_rx_queues - Configure hardware to start or stop the Rx queues
 * @vsi: VSI to enable/disable queues
 * @enable: true to enable queues, false to disable
 *
 * Control the Rx queues through the QRX_CTRL register, enabling or disabling
 * them. Wait for the appropriate time to ensure that the queues have actually
 * reached the expected state.
 */
int
ice_control_all_rx_queues(struct ice_vsi *vsi, bool enable)
{
	int i, err;

	/* TODO: amortize waits by changing all queues up front and then
	 * checking their status afterwards. This will become more necessary
	 * when we have a large number of queues.
	 */
	for (i = 0; i < vsi->num_rx_queues; i++) {
		err = ice_control_rx_queue(vsi, i, enable);
		if (err)
			break;
	}

	return (0);
}

/**
 * ice_configure_rxq_interrupt - Configure HW Rx queue for an MSI-X interrupt
 * @hw: ice hw structure
 * @rxqid: Rx queue index in PF space
 * @vector: MSI-X vector index in PF/VF space
 * @itr_idx: ITR index to use for interrupt
 *
 * @remark ice_flush() may need to be called after this
 */
void
ice_configure_rxq_interrupt(struct ice_hw *hw, uint16_t rxqid,
    uint16_t vector, uint8_t itr_idx)
{
	uint32_t val;

	KASSERT(itr_idx <= ICE_ITR_NONE);

	val = (QINT_RQCTL_CAUSE_ENA_M |
	       (itr_idx << QINT_RQCTL_ITR_INDX_S) |
	       (vector << QINT_RQCTL_MSIX_INDX_S));
	ICE_WRITE(hw, QINT_RQCTL(rxqid), val);
}

void
ice_configure_all_rxq_interrupts(struct ice_vsi *vsi)
{
	struct ice_hw *hw = &vsi->sc->hw;
	int i;

	for (i = 0; i < vsi->num_rx_queues; i++) {
		struct ice_rx_queue *rxq = &vsi->rx_queues[i];
		int v = rxq->irqv->iv_qid + 1;

		ice_configure_rxq_interrupt(hw, vsi->rx_qmap[rxq->me], v,
					    ICE_RX_ITR);

		DNPRINTF(ICE_DBG_INIT,
		    "%s: RXQ(%d) intr enable: me %d rxqid %d vector %d\n",
		    __func__, i, rxq->me, vsi->rx_qmap[rxq->me], v);
	}

	ice_flush(hw);
}

/**
 * ice_itr_to_reg - Convert an ITR setting into its register equivalent
 * @hw: The device HW structure
 * @itr_setting: the ITR setting to convert
 *
 * Based on the hardware ITR granularity, convert an ITR setting into the
 * correct value to prepare programming to the HW.
 */
static inline uint16_t ice_itr_to_reg(struct ice_hw *hw, uint16_t itr_setting)
{
	return itr_setting / hw->itr_gran;
}

/**
 * ice_configure_rx_itr - Configure the Rx ITR settings for this VSI
 * @vsi: the VSI to configure
 *
 * Program the hardware ITR registers with the settings for this VSI.
 */
void
ice_configure_rx_itr(struct ice_vsi *vsi)
{
	struct ice_hw *hw = &vsi->sc->hw;
	int i;

	/* TODO: Handle per-queue/per-vector ITR? */

	for (i = 0; i < vsi->num_rx_queues; i++) {
		struct ice_rx_queue *rxq = &vsi->rx_queues[i];
		int v = rxq->irqv->iv_qid + 1;

		ICE_WRITE(hw, GLINT_ITR(ICE_RX_ITR, v),
		     ice_itr_to_reg(hw, vsi->rx_itr));
	}

	ice_flush(hw);
}

/**
 * ice_set_default_promisc_mask - Set default config for promisc settings
 * @promisc_mask: bitmask to setup
 *
 * The ice_(set|clear)_vsi_promisc() function expects a mask of promiscuous
 * modes to operate on. The mask used in here is the default one for the
 * driver, where promiscuous is enabled/disabled for all types of
 * non-VLAN-tagged/VLAN 0 traffic.
 */
void
ice_set_default_promisc_mask(ice_bitmap_t *promisc_mask)
{
	ice_zero_bitmap(promisc_mask, ICE_PROMISC_MAX);
	ice_set_bit(ICE_PROMISC_UCAST_TX, promisc_mask);
	ice_set_bit(ICE_PROMISC_UCAST_RX, promisc_mask);
	ice_set_bit(ICE_PROMISC_MCAST_TX, promisc_mask);
	ice_set_bit(ICE_PROMISC_MCAST_RX, promisc_mask);
}

/**
 * _ice_set_vsi_promisc - set given VSI to given promiscuous mode(s)
 * @hw: pointer to the hardware structure
 * @vsi_handle: VSI handle to configure
 * @promisc_mask: pointer to mask of promiscuous config bits
 * @vid: VLAN ID to set VLAN promiscuous
 * @lport: logical port number to configure promisc mode
 */
enum ice_status
ice_set_vsi_promisc(struct ice_hw *hw, uint16_t vsi_handle,
		    ice_bitmap_t *promisc_mask, uint16_t vid)
{
	struct ice_switch_info *sw = hw->switch_info;
	uint8_t lport = hw->port_info->lport;
	enum { UCAST_FLTR = 1, MCAST_FLTR, BCAST_FLTR };
	ice_declare_bitmap(p_mask, ICE_PROMISC_MAX);
	struct ice_fltr_list_entry f_list_entry;
	struct ice_fltr_info new_fltr;
	enum ice_status status = ICE_SUCCESS;
	bool is_tx_fltr, is_rx_lb_fltr;
	uint16_t hw_vsi_id;
	int pkt_type;
	uint8_t recipe_id;

	DNPRINTF(ICE_DBG_TRACE, "%s\n", __func__);

	if (!ice_is_vsi_valid(hw, vsi_handle))
		return ICE_ERR_PARAM;
	hw_vsi_id = hw->vsi_ctx[vsi_handle]->vsi_num;

	memset(&new_fltr, 0, sizeof(new_fltr));

	/* Do not modify original bitmap */
	ice_cp_bitmap(p_mask, promisc_mask, ICE_PROMISC_MAX);

	if (ice_is_bit_set(p_mask, ICE_PROMISC_VLAN_RX) &&
	    ice_is_bit_set(p_mask, ICE_PROMISC_VLAN_TX)) {
		new_fltr.lkup_type = ICE_SW_LKUP_PROMISC_VLAN;
		new_fltr.l_data.mac_vlan.vlan_id = vid;
		recipe_id = ICE_SW_LKUP_PROMISC_VLAN;
	} else {
		new_fltr.lkup_type = ICE_SW_LKUP_PROMISC;
		recipe_id = ICE_SW_LKUP_PROMISC;
	}

	/* Separate filters must be set for each direction/packet type
	 * combination, so we will loop over the mask value, store the
	 * individual type, and clear it out in the input mask as it
	 * is found.
	 */
	while (ice_is_any_bit_set(p_mask, ICE_PROMISC_MAX)) {
		struct ice_sw_recipe *recp_list;
		uint8_t *mac_addr;

		pkt_type = 0;
		is_tx_fltr = false;
		is_rx_lb_fltr = false;

		if (ice_test_and_clear_bit(ICE_PROMISC_UCAST_RX,
					   p_mask)) {
			pkt_type = UCAST_FLTR;
		} else if (ice_test_and_clear_bit(ICE_PROMISC_UCAST_TX,
						  p_mask)) {
			pkt_type = UCAST_FLTR;
			is_tx_fltr = true;
		} else if (ice_test_and_clear_bit(ICE_PROMISC_MCAST_RX,
						  p_mask)) {
			pkt_type = MCAST_FLTR;
		} else if (ice_test_and_clear_bit(ICE_PROMISC_MCAST_TX,
						  p_mask)) {
			pkt_type = MCAST_FLTR;
			is_tx_fltr = true;
		} else if (ice_test_and_clear_bit(ICE_PROMISC_BCAST_RX,
						  p_mask)) {
			pkt_type = BCAST_FLTR;
		} else if (ice_test_and_clear_bit(ICE_PROMISC_BCAST_TX,
						  p_mask)) {
			pkt_type = BCAST_FLTR;
			is_tx_fltr = true;
		} else if (ice_test_and_clear_bit(ICE_PROMISC_UCAST_RX_LB,
						  p_mask)) {
			pkt_type = UCAST_FLTR;
			is_rx_lb_fltr = true;
		}

		/* Check for VLAN promiscuous flag */
		if (ice_is_bit_set(p_mask, ICE_PROMISC_VLAN_RX)) {
			ice_clear_bit(ICE_PROMISC_VLAN_RX, p_mask);
		} else if (ice_test_and_clear_bit(ICE_PROMISC_VLAN_TX,
						  p_mask)) {
			is_tx_fltr = true;
		}
		/* Set filter DA based on packet type */
		mac_addr = new_fltr.l_data.mac.mac_addr;
		if (pkt_type == BCAST_FLTR) {
			memset(mac_addr, 0xff, ETHER_ADDR_LEN);
		} else if (pkt_type == MCAST_FLTR ||
			   pkt_type == UCAST_FLTR) {
			/* Use the dummy ether header DA */
			memcpy(mac_addr, dummy_eth_header, ETHER_ADDR_LEN);
			if (pkt_type == MCAST_FLTR)
				mac_addr[0] |= 0x1;	/* Set multicast bit */
		}

		/* Need to reset this to zero for all iterations */
		new_fltr.flag = 0;
		if (is_tx_fltr) {
			new_fltr.flag |= ICE_FLTR_TX;
			new_fltr.src = hw_vsi_id;
		} else if (is_rx_lb_fltr) {
			new_fltr.flag |= ICE_FLTR_RX_LB;
			new_fltr.src = hw_vsi_id;
		} else {
			new_fltr.flag |= ICE_FLTR_RX;
			new_fltr.src = lport;
		}

		new_fltr.fltr_act = ICE_FWD_TO_VSI;
		new_fltr.vsi_handle = vsi_handle;
		new_fltr.fwd_id.hw_vsi_id = hw_vsi_id;
		f_list_entry.fltr_info = new_fltr;
		recp_list = &sw->recp_list[recipe_id];

		status = ice_add_rule_internal(hw, recp_list, lport,
					       &f_list_entry);
		if (status != ICE_SUCCESS)
			goto set_promisc_exit;
	}

set_promisc_exit:
	return status;
}

/**
 * ice_vsi_uses_fltr - Determine if given VSI uses specified filter
 * @fm_entry: filter entry to inspect
 * @vsi_handle: VSI handle to compare with filter info
 */
bool
ice_vsi_uses_fltr(struct ice_fltr_mgmt_list_entry *fm_entry,
    uint16_t vsi_handle)
{
	return ((fm_entry->fltr_info.fltr_act == ICE_FWD_TO_VSI &&
		 fm_entry->fltr_info.vsi_handle == vsi_handle) ||
		(fm_entry->fltr_info.fltr_act == ICE_FWD_TO_VSI_LIST &&
		 fm_entry->vsi_list_info &&
		 (ice_is_bit_set(fm_entry->vsi_list_info->vsi_map,
				 vsi_handle))));
}

/**
 * ice_add_entry_to_vsi_fltr_list - Add copy of fltr_list_entry to remove list
 * @hw: pointer to the hardware structure
 * @vsi_handle: VSI handle to remove filters from
 * @vsi_list_head: pointer to the list to add entry to
 * @fi: pointer to fltr_info of filter entry to copy & add
 *
 * Helper function, used when creating a list of filters to remove from
 * a specific VSI. The entry added to vsi_list_head is a COPY of the
 * original filter entry, with the exception of fltr_info.fltr_act and
 * fltr_info.fwd_id fields. These are set such that later logic can
 * extract which VSI to remove the fltr from, and pass on that information.
 */
enum ice_status
ice_add_entry_to_vsi_fltr_list(struct ice_hw *hw, uint16_t vsi_handle,
			       struct ice_fltr_list_head *vsi_list_head,
			       struct ice_fltr_info *fi)
{
	struct ice_fltr_list_entry *tmp;

	/* this memory is freed up in the caller function
	 * once filters for this VSI are removed
	 */
	tmp = (struct ice_fltr_list_entry *)ice_malloc(hw, sizeof(*tmp));
	if (!tmp)
		return ICE_ERR_NO_MEMORY;

	tmp->fltr_info = *fi;

	/* Overwrite these fields to indicate which VSI to remove filter from,
	 * so find and remove logic can extract the information from the
	 * list entries. Note that original entries will still have proper
	 * values.
	 */
	tmp->fltr_info.fltr_act = ICE_FWD_TO_VSI;
	tmp->fltr_info.vsi_handle = vsi_handle;
	tmp->fltr_info.fwd_id.hw_vsi_id = hw->vsi_ctx[vsi_handle]->vsi_num;

	TAILQ_INSERT_HEAD(vsi_list_head, tmp, list_entry);

	return ICE_SUCCESS;
}

/**
 * ice_determine_promisc_mask
 * @fi: filter info to parse
 * @promisc_mask: pointer to mask to be filled in
 *
 * Helper function to determine which ICE_PROMISC_ mask corresponds
 * to given filter into.
 */
void ice_determine_promisc_mask(struct ice_fltr_info *fi,
    ice_bitmap_t *promisc_mask)
{
	uint16_t vid = fi->l_data.mac_vlan.vlan_id;
	uint8_t *macaddr = fi->l_data.mac.mac_addr;
	bool is_rx_lb_fltr = false;
	bool is_tx_fltr = false;

	ice_zero_bitmap(promisc_mask, ICE_PROMISC_MAX);

	if (fi->flag == ICE_FLTR_TX)
		is_tx_fltr = true;
	if (fi->flag == ICE_FLTR_RX_LB)
		is_rx_lb_fltr = true;

	if (ETHER_IS_BROADCAST(macaddr)) {
		ice_set_bit(is_tx_fltr ? ICE_PROMISC_BCAST_TX
				       : ICE_PROMISC_BCAST_RX, promisc_mask);
	} else if (ETHER_IS_MULTICAST(macaddr)) {
		ice_set_bit(is_tx_fltr ? ICE_PROMISC_MCAST_TX
				       : ICE_PROMISC_MCAST_RX, promisc_mask);
	} else {
		if (is_tx_fltr)
			ice_set_bit(ICE_PROMISC_UCAST_TX, promisc_mask);
		else if (is_rx_lb_fltr)
			ice_set_bit(ICE_PROMISC_UCAST_RX_LB, promisc_mask);
		else
			ice_set_bit(ICE_PROMISC_UCAST_RX, promisc_mask);
	}

	if (vid) {
		ice_set_bit(is_tx_fltr ? ICE_PROMISC_VLAN_TX
				       : ICE_PROMISC_VLAN_RX, promisc_mask);
	}
}

/**
 * ice_remove_vsi_list_rule
 * @hw: pointer to the hardware structure
 * @vsi_list_id: VSI list ID generated as part of allocate resource
 * @lkup_type: switch rule filter lookup type
 *
 * The VSI list should be emptied before this function is called to remove the
 * VSI list.
 */
enum ice_status
ice_remove_vsi_list_rule(struct ice_hw *hw, uint16_t vsi_list_id,
			 enum ice_sw_lkup_type lkup_type)
{
	/* Free the vsi_list resource that we allocated. It is assumed that the
	 * list is empty at this point.
	 */
	return ice_aq_alloc_free_vsi_list(hw, &vsi_list_id, lkup_type,
	    ice_aqc_opc_free_res);
}

/**
 * ice_rem_update_vsi_list
 * @hw: pointer to the hardware structure
 * @vsi_handle: VSI handle of the VSI to remove
 * @fm_list: filter management entry for which the VSI list management needs to
 *           be done
 */
enum ice_status
ice_rem_update_vsi_list(struct ice_hw *hw, uint16_t vsi_handle,
			struct ice_fltr_mgmt_list_entry *fm_list)
{
	struct ice_switch_info *sw = hw->switch_info;
	enum ice_sw_lkup_type lkup_type;
	enum ice_status status = ICE_SUCCESS;
	uint16_t vsi_list_id;

	if (fm_list->fltr_info.fltr_act != ICE_FWD_TO_VSI_LIST ||
	    fm_list->vsi_count == 0)
		return ICE_ERR_PARAM;

	/* A rule with the VSI being removed does not exist */
	if (!ice_is_bit_set(fm_list->vsi_list_info->vsi_map, vsi_handle))
		return ICE_ERR_DOES_NOT_EXIST;

	lkup_type = fm_list->fltr_info.lkup_type;
	vsi_list_id = fm_list->fltr_info.fwd_id.vsi_list_id;
	status = ice_update_vsi_list_rule(hw, &vsi_handle, 1, vsi_list_id, true,
					  ice_aqc_opc_update_sw_rules,
					  lkup_type);
	if (status)
		return status;

	fm_list->vsi_count--;
	ice_clear_bit(vsi_handle, fm_list->vsi_list_info->vsi_map);

	if (fm_list->vsi_count == 1 && lkup_type != ICE_SW_LKUP_VLAN) {
		struct ice_fltr_info tmp_fltr_info = fm_list->fltr_info;
		struct ice_vsi_list_map_info *vsi_list_info =
			fm_list->vsi_list_info;
		uint16_t rem_vsi_handle;

		rem_vsi_handle = ice_find_first_bit(vsi_list_info->vsi_map,
						    ICE_MAX_VSI);
		if (!ice_is_vsi_valid(hw, rem_vsi_handle))
			return ICE_ERR_OUT_OF_RANGE;

		/* Make sure VSI list is empty before removing it below */
		status = ice_update_vsi_list_rule(hw, &rem_vsi_handle, 1,
						  vsi_list_id, true,
						  ice_aqc_opc_update_sw_rules,
						  lkup_type);
		if (status)
			return status;

		tmp_fltr_info.fltr_act = ICE_FWD_TO_VSI;
		tmp_fltr_info.fwd_id.hw_vsi_id =
			hw->vsi_ctx[rem_vsi_handle]->vsi_num;
		tmp_fltr_info.vsi_handle = rem_vsi_handle;
		status = ice_update_pkt_fwd_rule(hw, &tmp_fltr_info);
		if (status) {
			DNPRINTF(ICE_DBG_SW, "%s: Failed to update pkt fwd "
			    "rule to FWD_TO_VSI on HW VSI %d, error %d\n",
			    __func__,
			    tmp_fltr_info.fwd_id.hw_vsi_id, status);
			return status;
		}

		fm_list->fltr_info = tmp_fltr_info;
	}

	if ((fm_list->vsi_count == 1 && lkup_type != ICE_SW_LKUP_VLAN) ||
	    (fm_list->vsi_count == 0 && lkup_type == ICE_SW_LKUP_VLAN)) {
		struct ice_vsi_list_map_info *vsi_list_info =
			fm_list->vsi_list_info;

		/* Remove the VSI list since it is no longer used */
		status = ice_remove_vsi_list_rule(hw, vsi_list_id, lkup_type);
		if (status) {
			DNPRINTF(ICE_DBG_SW, "%s: Failed to remove "
			    "VSI list %d, error %d\n", __func__,
			    vsi_list_id, status);
			return status;
		}

		TAILQ_REMOVE(&sw->vsi_list_map_head, vsi_list_info, list_entry);
		ice_free(hw, vsi_list_info);
		fm_list->vsi_list_info = NULL;
	}

	return status;
}

/**
 * ice_remove_rule_internal - Remove a filter rule of a given type
 * @hw: pointer to the hardware structure
 * @recp_list: recipe list for which the rule needs to removed
 * @f_entry: rule entry containing filter information
 */
enum ice_status
ice_remove_rule_internal(struct ice_hw *hw, struct ice_sw_recipe *recp_list,
			 struct ice_fltr_list_entry *f_entry)
{
	struct ice_fltr_mgmt_list_entry *list_elem;
#if 0
	struct ice_lock *rule_lock; /* Lock to protect filter rule list */
#endif
	enum ice_status status = ICE_SUCCESS;
	bool remove_rule = false;
	uint16_t vsi_handle;

	if (!ice_is_vsi_valid(hw, f_entry->fltr_info.vsi_handle))
		return ICE_ERR_PARAM;
	f_entry->fltr_info.fwd_id.hw_vsi_id =
		hw->vsi_ctx[f_entry->fltr_info.vsi_handle]->vsi_num;
#if 0
	rule_lock = &recp_list->filt_rule_lock;
	ice_acquire_lock(rule_lock);
#endif
	list_elem = ice_find_rule_entry(&recp_list->filt_rules,
					&f_entry->fltr_info);
	if (!list_elem) {
		status = ICE_ERR_DOES_NOT_EXIST;
		goto exit;
	}

	if (list_elem->fltr_info.fltr_act != ICE_FWD_TO_VSI_LIST) {
		remove_rule = true;
	} else if (!list_elem->vsi_list_info) {
		status = ICE_ERR_DOES_NOT_EXIST;
		goto exit;
	} else if (list_elem->vsi_list_info->ref_cnt > 1) {
		/* a ref_cnt > 1 indicates that the vsi_list is being
		 * shared by multiple rules. Decrement the ref_cnt and
		 * remove this rule, but do not modify the list, as it
		 * is in-use by other rules.
		 */
		list_elem->vsi_list_info->ref_cnt--;
		remove_rule = true;
	} else {
		/* a ref_cnt of 1 indicates the vsi_list is only used
		 * by one rule. However, the original removal request is only
		 * for a single VSI. Update the vsi_list first, and only
		 * remove the rule if there are no further VSIs in this list.
		 */
		vsi_handle = f_entry->fltr_info.vsi_handle;
		status = ice_rem_update_vsi_list(hw, vsi_handle, list_elem);
		if (status)
			goto exit;
		/* if VSI count goes to zero after updating the VSI list */
		if (list_elem->vsi_count == 0)
			remove_rule = true;
	}

	if (remove_rule) {
		/* Remove the lookup rule */
		struct ice_sw_rule_lkup_rx_tx *s_rule;

		s_rule = (struct ice_sw_rule_lkup_rx_tx *)
			ice_malloc(hw, ice_struct_size(s_rule, hdr_data, 0));
		if (!s_rule) {
			status = ICE_ERR_NO_MEMORY;
			goto exit;
		}

		ice_fill_sw_rule(hw, &list_elem->fltr_info, s_rule,
				 ice_aqc_opc_remove_sw_rules);

		status = ice_aq_sw_rules(hw, s_rule,
					 ice_struct_size(s_rule, hdr_data, 0),
					 1, ice_aqc_opc_remove_sw_rules, NULL);

		/* Remove a book keeping from the list */
		ice_free(hw, s_rule);

		if (status)
			goto exit;

		TAILQ_REMOVE(&recp_list->filt_rules, list_elem, list_entry);
		ice_free(hw, list_elem);
	}
exit:
#if 0
	ice_release_lock(rule_lock);
#endif
	return status;
}

/**
 * ice_remove_promisc - Remove promisc based filter rules
 * @hw: pointer to the hardware structure
 * @recp_id: recipe ID for which the rule needs to removed
 * @v_list: list of promisc entries
 */
enum ice_status
ice_remove_promisc(struct ice_hw *hw, uint8_t recp_id,
		   struct ice_fltr_list_head *v_list)
{
	struct ice_fltr_list_entry *v_list_itr, *tmp;
	struct ice_sw_recipe *recp_list;

	recp_list = &hw->switch_info->recp_list[recp_id];
	TAILQ_FOREACH_SAFE(v_list_itr, v_list, list_entry, tmp) {
		v_list_itr->status =
			ice_remove_rule_internal(hw, recp_list, v_list_itr);
		if (v_list_itr->status)
			return v_list_itr->status;
	}
	return ICE_SUCCESS;
}
/**
 * ice_clear_vsi_promisc - clear specified promiscuous mode(s)
 * @hw: pointer to the hardware structure
 * @vsi_handle: VSI handle to clear mode
 * @promisc_mask: pointer to mask of promiscuous config bits to clear
 * @vid: VLAN ID to clear VLAN promiscuous
 */
enum ice_status
ice_clear_vsi_promisc(struct ice_hw *hw, uint16_t vsi_handle,
		       ice_bitmap_t *promisc_mask, uint16_t vid)
{
	struct ice_switch_info *sw = hw->switch_info;
	ice_declare_bitmap(compl_promisc_mask, ICE_PROMISC_MAX);
	ice_declare_bitmap(fltr_promisc_mask, ICE_PROMISC_MAX);
	struct ice_fltr_list_entry *fm_entry, *tmp;
	struct ice_fltr_list_head remove_list_head;
	struct ice_fltr_mgmt_list_entry *itr;
	struct ice_fltr_mgmt_list_head *rule_head;
#if 0
	struct ice_lock *rule_lock;	/* Lock to protect filter rule list */
#endif
	enum ice_status status = ICE_SUCCESS;
	uint8_t recipe_id;

	if (!ice_is_vsi_valid(hw, vsi_handle))
		return ICE_ERR_PARAM;

	if (ice_is_bit_set(promisc_mask, ICE_PROMISC_VLAN_RX) &&
	    ice_is_bit_set(promisc_mask, ICE_PROMISC_VLAN_TX))
		recipe_id = ICE_SW_LKUP_PROMISC_VLAN;
	else
		recipe_id = ICE_SW_LKUP_PROMISC;

	rule_head = &sw->recp_list[recipe_id].filt_rules;
#if 0
	rule_lock = &sw->recp_list[recipe_id].filt_rule_lock;
#endif
	TAILQ_INIT(&remove_list_head);
#if 0
	ice_acquire_lock(rule_lock);
#endif
	TAILQ_FOREACH(itr, rule_head, list_entry) {
		struct ice_fltr_info *fltr_info;
		ice_zero_bitmap(compl_promisc_mask, ICE_PROMISC_MAX);

		if (!ice_vsi_uses_fltr(itr, vsi_handle))
			continue;
		fltr_info = &itr->fltr_info;

		if (recipe_id == ICE_SW_LKUP_PROMISC_VLAN &&
		    vid != fltr_info->l_data.mac_vlan.vlan_id)
			continue;

		ice_determine_promisc_mask(fltr_info, fltr_promisc_mask);
		ice_andnot_bitmap(compl_promisc_mask, fltr_promisc_mask,
				  promisc_mask, ICE_PROMISC_MAX);

		/* Skip if filter is not completely specified by given mask */
		if (ice_is_any_bit_set(compl_promisc_mask, ICE_PROMISC_MAX))
			continue;

		status = ice_add_entry_to_vsi_fltr_list(hw, vsi_handle,
							&remove_list_head,
							fltr_info);
		if (status) {
#if 0
			ice_release_lock(rule_lock);
#endif
			goto free_fltr_list;
		}
	}
#if 0
	ice_release_lock(rule_lock);
#endif
	status = ice_remove_promisc(hw, recipe_id, &remove_list_head);

free_fltr_list:
	TAILQ_FOREACH_SAFE(fm_entry, &remove_list_head, list_entry, tmp) {
		TAILQ_REMOVE(&remove_list_head, fm_entry, list_entry);
		ice_free(hw, fm_entry);
	}

	return status;
}

/**
 * ice_if_promisc_set - Set device promiscuous mode
 *
 * @remark Calls to this function will always overwrite the previous setting
 */
int
ice_if_promisc_set(struct ice_softc *sc)
{
	struct ifnet *ifp = &sc->sc_ac.ac_if;
	struct ice_hw *hw = &sc->hw;
	enum ice_status status;
	bool promisc_enable = ifp->if_flags & IFF_PROMISC;
	bool multi_enable = ifp->if_flags & IFF_ALLMULTI;
	ice_declare_bitmap(promisc_mask, ICE_PROMISC_MAX);

	/* Do not support configuration when in recovery mode */
	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
		return (ENOSYS);
	
	ice_set_default_promisc_mask(promisc_mask);

	if (multi_enable)
		return (EOPNOTSUPP);

	if (promisc_enable) {
		status = ice_set_vsi_promisc(hw, sc->pf_vsi.idx,
					     promisc_mask, 0);
		if (status && status != ICE_ERR_ALREADY_EXISTS) {
			printf("%s: Failed to enable promiscuous mode for "
			    "PF VSI, err %s aq_err %s\n",
			    sc->sc_dev.dv_xname,
			    ice_status_str(status),
			    ice_aq_str(hw->adminq.sq_last_status));
			return (EIO);
		}
	} else {
		status = ice_clear_vsi_promisc(hw, sc->pf_vsi.idx,
					       promisc_mask, 0);
		if (status) {
			printf("%s: Failed to disable promiscuous mode for "
			    "PF VSI, err %s aq_err %s\n",
			    sc->sc_dev.dv_xname,
			    ice_status_str(status),
			    ice_aq_str(hw->adminq.sq_last_status));
			return (EIO);
		}
	}

	return (0);
}

/**
 * ice_enable_intr - Enable interrupts for given vector
 * @hw: the device private HW structure
 * @vector: the interrupt index in PF space
 *
 * In MSI or Legacy interrupt mode, interrupt 0 is the only valid index.
 */
void
ice_enable_intr(struct ice_hw *hw, int vector)
{
	uint32_t dyn_ctl;

	/* Use ITR_NONE so that ITR configuration is not changed. */
	dyn_ctl = GLINT_DYN_CTL_INTENA_M | GLINT_DYN_CTL_CLEARPBA_M |
		  (ICE_ITR_NONE << GLINT_DYN_CTL_ITR_INDX_S);
	ICE_WRITE(hw, GLINT_DYN_CTL(vector), dyn_ctl);
}

/**
 * ice_copy_phy_caps_to_cfg - Copy PHY ability data to configuration data
 * @pi: port information structure
 * @caps: PHY ability structure to copy data from
 * @cfg: PHY configuration structure to copy data to
 *
 * Helper function to copy AQC PHY get ability data to PHY set configuration
 * data structure
 */
void
ice_copy_phy_caps_to_cfg(struct ice_port_info *pi,
			 struct ice_aqc_get_phy_caps_data *caps,
			 struct ice_aqc_set_phy_cfg_data *cfg)
{
	if (!pi || !caps || !cfg)
		return;

	memset(cfg, 0, sizeof(*cfg));
	cfg->phy_type_low = caps->phy_type_low;
	cfg->phy_type_high = caps->phy_type_high;
	cfg->caps = caps->caps;
	cfg->low_power_ctrl_an = caps->low_power_ctrl_an;
	cfg->eee_cap = caps->eee_cap;
	cfg->eeer_value = caps->eeer_value;
	cfg->link_fec_opt = caps->link_fec_options;
	cfg->module_compliance_enforcement =
		caps->module_compliance_enforcement;
}

#define ICE_PHYS_100MB			\
    (ICE_PHY_TYPE_LOW_100BASE_TX |	\
     ICE_PHY_TYPE_LOW_100M_SGMII)
#define ICE_PHYS_1000MB			\
    (ICE_PHY_TYPE_LOW_1000BASE_T |	\
     ICE_PHY_TYPE_LOW_1000BASE_SX |	\
     ICE_PHY_TYPE_LOW_1000BASE_LX |	\
     ICE_PHY_TYPE_LOW_1000BASE_KX |	\
     ICE_PHY_TYPE_LOW_1G_SGMII)
#define ICE_PHYS_2500MB			\
    (ICE_PHY_TYPE_LOW_2500BASE_T |	\
     ICE_PHY_TYPE_LOW_2500BASE_X |	\
     ICE_PHY_TYPE_LOW_2500BASE_KX)
#define ICE_PHYS_5GB			\
    (ICE_PHY_TYPE_LOW_5GBASE_T |	\
     ICE_PHY_TYPE_LOW_5GBASE_KR)
#define ICE_PHYS_10GB			\
    (ICE_PHY_TYPE_LOW_10GBASE_T |	\
     ICE_PHY_TYPE_LOW_10G_SFI_DA |	\
     ICE_PHY_TYPE_LOW_10GBASE_SR |	\
     ICE_PHY_TYPE_LOW_10GBASE_LR |	\
     ICE_PHY_TYPE_LOW_10GBASE_KR_CR1 |	\
     ICE_PHY_TYPE_LOW_10G_SFI_AOC_ACC |	\
     ICE_PHY_TYPE_LOW_10G_SFI_C2C)
#define ICE_PHYS_25GB			\
    (ICE_PHY_TYPE_LOW_25GBASE_T |	\
     ICE_PHY_TYPE_LOW_25GBASE_CR |	\
     ICE_PHY_TYPE_LOW_25GBASE_CR_S |	\
     ICE_PHY_TYPE_LOW_25GBASE_CR1 |	\
     ICE_PHY_TYPE_LOW_25GBASE_SR |	\
     ICE_PHY_TYPE_LOW_25GBASE_LR |	\
     ICE_PHY_TYPE_LOW_25GBASE_KR |	\
     ICE_PHY_TYPE_LOW_25GBASE_KR_S |	\
     ICE_PHY_TYPE_LOW_25GBASE_KR1 |	\
     ICE_PHY_TYPE_LOW_25G_AUI_AOC_ACC |	\
     ICE_PHY_TYPE_LOW_25G_AUI_C2C)
#define ICE_PHYS_40GB			\
    (ICE_PHY_TYPE_LOW_40GBASE_CR4 |	\
     ICE_PHY_TYPE_LOW_40GBASE_SR4 |	\
     ICE_PHY_TYPE_LOW_40GBASE_LR4 |	\
     ICE_PHY_TYPE_LOW_40GBASE_KR4 |	\
     ICE_PHY_TYPE_LOW_40G_XLAUI_AOC_ACC | \
     ICE_PHY_TYPE_LOW_40G_XLAUI)
#define ICE_PHYS_50GB			\
    (ICE_PHY_TYPE_LOW_50GBASE_CR2 |	\
     ICE_PHY_TYPE_LOW_50GBASE_SR2 |	\
     ICE_PHY_TYPE_LOW_50GBASE_LR2 |	\
     ICE_PHY_TYPE_LOW_50GBASE_KR2 |	\
     ICE_PHY_TYPE_LOW_50G_LAUI2_AOC_ACC | \
     ICE_PHY_TYPE_LOW_50G_LAUI2 |	\
     ICE_PHY_TYPE_LOW_50G_AUI2_AOC_ACC | \
     ICE_PHY_TYPE_LOW_50G_AUI2 |	\
     ICE_PHY_TYPE_LOW_50GBASE_CP |	\
     ICE_PHY_TYPE_LOW_50GBASE_SR |	\
     ICE_PHY_TYPE_LOW_50GBASE_FR |	\
     ICE_PHY_TYPE_LOW_50GBASE_LR |	\
     ICE_PHY_TYPE_LOW_50GBASE_KR_PAM4 |	\
     ICE_PHY_TYPE_LOW_50G_AUI1_AOC_ACC | \
     ICE_PHY_TYPE_LOW_50G_AUI1)
#define ICE_PHYS_100GB_LOW		\
    (ICE_PHY_TYPE_LOW_100GBASE_CR4 |	\
     ICE_PHY_TYPE_LOW_100GBASE_SR4 |	\
     ICE_PHY_TYPE_LOW_100GBASE_LR4 |	\
     ICE_PHY_TYPE_LOW_100GBASE_KR4 |	\
     ICE_PHY_TYPE_LOW_100G_CAUI4_AOC_ACC | \
     ICE_PHY_TYPE_LOW_100G_CAUI4 |	\
     ICE_PHY_TYPE_LOW_100G_AUI4_AOC_ACC | \
     ICE_PHY_TYPE_LOW_100G_AUI4 |	\
     ICE_PHY_TYPE_LOW_100GBASE_CR_PAM4 | \
     ICE_PHY_TYPE_LOW_100GBASE_KR_PAM4 | \
     ICE_PHY_TYPE_LOW_100GBASE_CP2 |	\
     ICE_PHY_TYPE_LOW_100GBASE_SR2 |	\
     ICE_PHY_TYPE_LOW_100GBASE_DR)
#define ICE_PHYS_100GB_HIGH		\
    (ICE_PHY_TYPE_HIGH_100GBASE_KR2_PAM4 | \
     ICE_PHY_TYPE_HIGH_100G_CAUI2_AOC_ACC | \
     ICE_PHY_TYPE_HIGH_100G_CAUI2 |	\
     ICE_PHY_TYPE_HIGH_100G_AUI2_AOC_ACC | \
     ICE_PHY_TYPE_HIGH_100G_AUI2)

/**
 * ice_sysctl_speeds_to_aq_phy_types - Convert sysctl speed flags to AQ PHY flags
 * @sysctl_speeds: 16-bit sysctl speeds or AQ_LINK_SPEED flags
 * @phy_type_low: output parameter for lower AQ PHY flags
 * @phy_type_high: output parameter for higher AQ PHY flags
 *
 * Converts the given link speed flags into AQ PHY type flag sets appropriate
 * for use in a Set PHY Config command.
 */
void
ice_sysctl_speeds_to_aq_phy_types(uint16_t sysctl_speeds,
    uint64_t *phy_type_low, uint64_t *phy_type_high)
{
	*phy_type_low = 0, *phy_type_high = 0;

	if (sysctl_speeds & ICE_AQ_LINK_SPEED_100MB)
		*phy_type_low |= ICE_PHYS_100MB;
	if (sysctl_speeds & ICE_AQ_LINK_SPEED_1000MB)
		*phy_type_low |= ICE_PHYS_1000MB;
	if (sysctl_speeds & ICE_AQ_LINK_SPEED_2500MB)
		*phy_type_low |= ICE_PHYS_2500MB;
	if (sysctl_speeds & ICE_AQ_LINK_SPEED_5GB)
		*phy_type_low |= ICE_PHYS_5GB;
	if (sysctl_speeds & ICE_AQ_LINK_SPEED_10GB)
		*phy_type_low |= ICE_PHYS_10GB;
	if (sysctl_speeds & ICE_AQ_LINK_SPEED_25GB)
		*phy_type_low |= ICE_PHYS_25GB;
	if (sysctl_speeds & ICE_AQ_LINK_SPEED_40GB)
		*phy_type_low |= ICE_PHYS_40GB;
	if (sysctl_speeds & ICE_AQ_LINK_SPEED_50GB)
		*phy_type_low |= ICE_PHYS_50GB;
	if (sysctl_speeds & ICE_AQ_LINK_SPEED_100GB) {
		*phy_type_low |= ICE_PHYS_100GB_LOW;
		*phy_type_high |= ICE_PHYS_100GB_HIGH;
	}
}

/**
 * @struct ice_phy_data
 * @brief PHY caps and link speeds
 *
 * Buffer providing report mode and user speeds;
 * returning intersection of PHY types and speeds.
 */
struct ice_phy_data {
	uint64_t phy_low_orig;     /* PHY low quad from report */
	uint64_t phy_high_orig;    /* PHY high quad from report */
	uint64_t phy_low_intr;     /* low quad intersection with user speeds */
	uint64_t phy_high_intr;    /* high quad intersection with user speeds */
	uint16_t user_speeds_orig; /* Input from caller - ICE_AQ_LINK_SPEED_* */
	uint16_t user_speeds_intr; /* Intersect with report speeds */
	uint8_t report_mode;       /* See ICE_AQC_REPORT_* */
};

/**
 * @var phy_link_speeds
 * @brief PHY link speed conversion array
 *
 * Array of link speeds to convert ICE_PHY_TYPE_LOW and ICE_PHY_TYPE_HIGH into
 * link speeds used by the link speed sysctls.
 *
 * @remark these are based on the indices used in the BIT() macros for the
 * ICE_PHY_TYPE_LOW_* and ICE_PHY_TYPE_HIGH_* definitions.
 */
static const uint16_t phy_link_speeds[] = {
    ICE_AQ_LINK_SPEED_100MB,
    ICE_AQ_LINK_SPEED_100MB,
    ICE_AQ_LINK_SPEED_1000MB,
    ICE_AQ_LINK_SPEED_1000MB,
    ICE_AQ_LINK_SPEED_1000MB,
    ICE_AQ_LINK_SPEED_1000MB,
    ICE_AQ_LINK_SPEED_1000MB,
    ICE_AQ_LINK_SPEED_2500MB,
    ICE_AQ_LINK_SPEED_2500MB,
    ICE_AQ_LINK_SPEED_2500MB,
    ICE_AQ_LINK_SPEED_5GB,
    ICE_AQ_LINK_SPEED_5GB,
    ICE_AQ_LINK_SPEED_10GB,
    ICE_AQ_LINK_SPEED_10GB,
    ICE_AQ_LINK_SPEED_10GB,
    ICE_AQ_LINK_SPEED_10GB,
    ICE_AQ_LINK_SPEED_10GB,
    ICE_AQ_LINK_SPEED_10GB,
    ICE_AQ_LINK_SPEED_10GB,
    ICE_AQ_LINK_SPEED_25GB,
    ICE_AQ_LINK_SPEED_25GB,
    ICE_AQ_LINK_SPEED_25GB,
    ICE_AQ_LINK_SPEED_25GB,
    ICE_AQ_LINK_SPEED_25GB,
    ICE_AQ_LINK_SPEED_25GB,
    ICE_AQ_LINK_SPEED_25GB,
    ICE_AQ_LINK_SPEED_25GB,
    ICE_AQ_LINK_SPEED_25GB,
    ICE_AQ_LINK_SPEED_25GB,
    ICE_AQ_LINK_SPEED_25GB,
    ICE_AQ_LINK_SPEED_40GB,
    ICE_AQ_LINK_SPEED_40GB,
    ICE_AQ_LINK_SPEED_40GB,
    ICE_AQ_LINK_SPEED_40GB,
    ICE_AQ_LINK_SPEED_40GB,
    ICE_AQ_LINK_SPEED_40GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    /* These rates are for ICE_PHY_TYPE_HIGH_* */
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB
};

/**
 * ice_aq_phy_types_to_link_speeds - Convert the PHY Types to speeds
 * @phy_type_low: lower 64-bit PHY Type bitmask
 * @phy_type_high: upper 64-bit PHY Type bitmask
 *
 * Convert the PHY Type fields from Get PHY Abilities and Set PHY Config into
 * link speed flags. If phy_type_high has an unknown PHY type, then the return
 * value will include the "ICE_AQ_LINK_SPEED_UNKNOWN" flag as well.
 */
uint16_t
ice_aq_phy_types_to_link_speeds(uint64_t phy_type_low, uint64_t phy_type_high)
{
	uint16_t sysctl_speeds = 0;
	int bit;

	for (bit = 0; bit < 64; bit++) {
		if (phy_type_low & (1ULL << bit))
			sysctl_speeds |= phy_link_speeds[bit];
	}

	for (bit = 0; bit < 64; bit++) {
		if ((phy_type_high & (1ULL << bit)) == 0)
			continue;
		if ((bit + 64) < (int)nitems(phy_link_speeds))
			sysctl_speeds |= phy_link_speeds[bit + 64];
		else
			sysctl_speeds |= ICE_AQ_LINK_SPEED_UNKNOWN;
	}

	return (sysctl_speeds);
}

/**
 * ice_apply_supported_speed_filter - Mask off unsupported speeds
 * @report_speeds: bit-field for the desired link speeds
 * @mod_type: type of module/sgmii connection we have
 *
 * Given a bitmap of the desired lenient mode link speeds,
 * this function will mask off the speeds that are not currently
 * supported by the device.
 */
uint16_t
ice_apply_supported_speed_filter(uint16_t report_speeds, uint8_t mod_type)
{
	uint16_t speed_mask;
	enum { IS_SGMII, IS_SFP, IS_QSFP } module;

	/*
	 * The SFF specification says 0 is unknown, so we'll
	 * treat it like we're connected through SGMII for now.
	 * This may need revisiting if a new type is supported
	 * in the future.
	 */
	switch (mod_type) {
	case 0:
		module = IS_SGMII;
		break;
	case 3:
		module = IS_SFP;
		break;
	default:
		module = IS_QSFP;
		break;
	}

	/* We won't offer anything lower than 100M for any part,
	 * but we'll need to mask off other speeds based on the
	 * device and module type.
	 */
	speed_mask = ~((uint16_t)ICE_AQ_LINK_SPEED_100MB - 1);
	if ((report_speeds & ICE_AQ_LINK_SPEED_10GB) && (module == IS_SFP))
		speed_mask = ~((uint16_t)ICE_AQ_LINK_SPEED_1000MB - 1);
	if (report_speeds & ICE_AQ_LINK_SPEED_25GB)
		speed_mask = ~((uint16_t)ICE_AQ_LINK_SPEED_1000MB - 1);
	if (report_speeds & ICE_AQ_LINK_SPEED_50GB) {
		speed_mask = ~((uint16_t)ICE_AQ_LINK_SPEED_1000MB - 1);
		if (module == IS_QSFP)
			speed_mask = ~((uint16_t)ICE_AQ_LINK_SPEED_10GB - 1);
	}
	if (report_speeds & ICE_AQ_LINK_SPEED_100GB)
		speed_mask = ~((uint16_t)ICE_AQ_LINK_SPEED_25GB - 1);
	return (report_speeds & speed_mask);
}

/**
 * ice_intersect_phy_types_and_speeds - Return intersection of link speeds
 * @sc: device private structure
 * @phy_data: device PHY data
 *
 * On read: Displays the currently supported speeds
 * On write: Sets the device's supported speeds
 * Valid input flags: see ICE_SYSCTL_HELP_ADVERTISE_SPEED
 */
int
ice_intersect_phy_types_and_speeds(struct ice_softc *sc,
				   struct ice_phy_data *phy_data)
{
	struct ice_aqc_get_phy_caps_data pcaps = { 0 };
	const char *report_types[5] = { "w/o MEDIA",
					"w/MEDIA",
					"ACTIVE",
					"EDOOFUS", /* Not used */
					"DFLT" };
	struct ice_hw *hw = &sc->hw;
	struct ice_port_info *pi = hw->port_info;
	enum ice_status status;
	uint16_t report_speeds, temp_speeds;
	uint8_t report_type;
	bool apply_speed_filter = false;

	switch (phy_data->report_mode) {
	case ICE_AQC_REPORT_TOPO_CAP_NO_MEDIA:
	case ICE_AQC_REPORT_TOPO_CAP_MEDIA:
	case ICE_AQC_REPORT_ACTIVE_CFG:
	case ICE_AQC_REPORT_DFLT_CFG:
		report_type = phy_data->report_mode >> 1;
		break;
	default:
		DPRINTF("%s: phy_data.report_mode \"%u\" doesn't exist\n",
		    __func__, phy_data->report_mode);
		return (EINVAL);
	}

	/* 0 is treated as "Auto"; the driver will handle selecting the
	 * correct speeds. Including, in some cases, applying an override
	 * if provided.
	 */
	if (phy_data->user_speeds_orig == 0)
		phy_data->user_speeds_orig = USHRT_MAX;
	else if (ice_is_bit_set(sc->feat_en, ICE_FEATURE_LENIENT_LINK_MODE))
		apply_speed_filter = true;

	status = ice_aq_get_phy_caps(pi, false, phy_data->report_mode, &pcaps, NULL);
	if (status != ICE_SUCCESS) {
		printf("%s: ice_aq_get_phy_caps (%s) failed; status %s, "
		    "aq_err %s\n", sc->sc_dev.dv_xname,
		    report_types[report_type], ice_status_str(status),
		    ice_aq_str(sc->hw.adminq.sq_last_status));
		return (EIO);
	}

	phy_data->phy_low_orig = le64toh(pcaps.phy_type_low);
	phy_data->phy_high_orig = le64toh(pcaps.phy_type_high);
	report_speeds = ice_aq_phy_types_to_link_speeds(phy_data->phy_low_orig,
	    phy_data->phy_high_orig);
	if (apply_speed_filter) {
		temp_speeds = ice_apply_supported_speed_filter(report_speeds,
		    pcaps.module_type[0]);
		if ((phy_data->user_speeds_orig & temp_speeds) == 0) {
			printf("%s: User-specified speeds (\"0x%04X\") not "
			    "supported\n", sc->sc_dev.dv_xname,
			    phy_data->user_speeds_orig);
			return (EINVAL);
		}
		report_speeds = temp_speeds;
	}
	ice_sysctl_speeds_to_aq_phy_types(phy_data->user_speeds_orig,
	    &phy_data->phy_low_intr, &phy_data->phy_high_intr);
	phy_data->user_speeds_intr = phy_data->user_speeds_orig & report_speeds;
	phy_data->phy_low_intr &= phy_data->phy_low_orig;
	phy_data->phy_high_intr &= phy_data->phy_high_orig;

	return (0);
 }

/**
 * ice_apply_saved_phy_req_to_cfg -- Write saved user PHY settings to cfg data
 * @sc: device private structure
 * @cfg: new PHY config data to be modified
 *
 * Applies user settings for advertised speeds to the PHY type fields in the
 * supplied PHY config struct. It uses the data from pcaps to check if the
 * saved settings are invalid and uses the pcaps data instead if they are
 * invalid.
 */
int
ice_apply_saved_phy_req_to_cfg(struct ice_softc *sc,
			       struct ice_aqc_set_phy_cfg_data *cfg)
{
	struct ice_phy_data phy_data = { 0 };
	struct ice_port_info *pi = sc->hw.port_info;
	uint64_t phy_low = 0, phy_high = 0;
	uint16_t link_speeds;
	int ret;

	link_speeds = pi->phy.curr_user_speed_req;
	if (ice_is_bit_set(sc->feat_en, ICE_FEATURE_LINK_MGMT_VER_2)) {
		memset(&phy_data, 0, sizeof(phy_data));
		phy_data.report_mode = ICE_AQC_REPORT_DFLT_CFG;
		phy_data.user_speeds_orig = link_speeds;
		ret = ice_intersect_phy_types_and_speeds(sc, &phy_data);
		if (ret != 0)
			return (ret);
		phy_low = phy_data.phy_low_intr;
		phy_high = phy_data.phy_high_intr;

		if (link_speeds == 0 || phy_data.user_speeds_intr)
			goto finalize_link_speed;
		if (ice_is_bit_set(sc->feat_en,
		    ICE_FEATURE_LENIENT_LINK_MODE)) {
			memset(&phy_data, 0, sizeof(phy_data));
			phy_data.report_mode = ICE_AQC_REPORT_TOPO_CAP_NO_MEDIA;
			phy_data.user_speeds_orig = link_speeds;
			ret = ice_intersect_phy_types_and_speeds(sc, &phy_data);
			if (ret != 0)
				return (ret);
			phy_low = phy_data.phy_low_intr;
			phy_high = phy_data.phy_high_intr;

			if (!phy_data.user_speeds_intr) {
				phy_low = phy_data.phy_low_orig;
				phy_high = phy_data.phy_high_orig;
			}
			goto finalize_link_speed;
		}
		/* If we're here, then it means the benefits of Version 2
		 * link management aren't utilized.  We fall through to
		 * handling Strict Link Mode the same as Version 1 link
		 * management.
		 */
	}

	memset(&phy_data, 0, sizeof(phy_data));
	if ((link_speeds == 0) &&
	    (sc->ldo_tlv.phy_type_low || sc->ldo_tlv.phy_type_high))
		phy_data.report_mode = ICE_AQC_REPORT_TOPO_CAP_NO_MEDIA;
	else
		phy_data.report_mode = ICE_AQC_REPORT_TOPO_CAP_MEDIA;
	phy_data.user_speeds_orig = link_speeds;
	ret = ice_intersect_phy_types_and_speeds(sc, &phy_data);
	if (ret != 0)
		return (ret);
	phy_low = phy_data.phy_low_intr;
	phy_high = phy_data.phy_high_intr;

	if (!ice_is_bit_set(sc->feat_en, ICE_FEATURE_LENIENT_LINK_MODE)) {
		if (phy_low == 0 && phy_high == 0) {
			printf("%s: The selected speed is not supported by "
			    "the current media. Please select a link speed "
			    "that is supported by the current media.\n",
			    sc->sc_dev.dv_xname);
			return (EINVAL);
		}
	} else {
		if (link_speeds == 0) {
			if (sc->ldo_tlv.phy_type_low & phy_low ||
			    sc->ldo_tlv.phy_type_high & phy_high) {
				phy_low &= sc->ldo_tlv.phy_type_low;
				phy_high &= sc->ldo_tlv.phy_type_high;
			}
		} else if (phy_low == 0 && phy_high == 0) {
			memset(&phy_data, 0, sizeof(phy_data));
			phy_data.report_mode = ICE_AQC_REPORT_TOPO_CAP_NO_MEDIA;
			phy_data.user_speeds_orig = link_speeds;
			ret = ice_intersect_phy_types_and_speeds(sc, &phy_data);
			if (ret != 0)
				return (ret);
			phy_low = phy_data.phy_low_intr;
			phy_high = phy_data.phy_high_intr;

			if (!phy_data.user_speeds_intr) {
				phy_low = phy_data.phy_low_orig;
				phy_high = phy_data.phy_high_orig;
			}
		}
	}

finalize_link_speed:
	/* Cache new user settings for speeds */
	pi->phy.curr_user_speed_req = phy_data.user_speeds_intr;
	cfg->phy_type_low = htole64(phy_low);
	cfg->phy_type_high = htole64(phy_high);

	return (ret);
}

/**
 * ice_get_pfa_module_tlv - Reads sub module TLV from NVM PFA
 * @hw: pointer to hardware structure
 * @module_tlv: pointer to module TLV to return
 * @module_tlv_len: pointer to module TLV length to return
 * @module_type: module type requested
 *
 * Finds the requested sub module TLV type from the Preserved Field
 * Area (PFA) and returns the TLV pointer and length. The caller can
 * use these to read the variable length TLV value.
 */
enum ice_status
ice_get_pfa_module_tlv(struct ice_hw *hw, uint16_t *module_tlv,
    uint16_t *module_tlv_len, uint16_t module_type)
{
	enum ice_status status;
	uint16_t pfa_len, pfa_ptr;
	uint16_t next_tlv;

	status = ice_read_sr_word(hw, ICE_SR_PFA_PTR, &pfa_ptr);
	if (status != ICE_SUCCESS) {
		DNPRINTF(ICE_DBG_INIT, "%s: Preserved Field Array pointer.\n",
		    __func__);
		return status;
	}
	status = ice_read_sr_word(hw, pfa_ptr, &pfa_len);
	if (status != ICE_SUCCESS) {
		DNPRINTF(ICE_DBG_INIT, "%s: Failed to read PFA length.\n",
		    __func__);
		return status;
	}
	/* Starting with first TLV after PFA length, iterate through the list
	 * of TLVs to find the requested one.
	 */
	next_tlv = pfa_ptr + 1;
	while (next_tlv < pfa_ptr + pfa_len) {
		uint16_t tlv_sub_module_type;
		uint16_t tlv_len;

		/* Read TLV type */
		status = ice_read_sr_word(hw, next_tlv, &tlv_sub_module_type);
		if (status != ICE_SUCCESS) {
			DNPRINTF(ICE_DBG_INIT, "%s: Failed to read TLV type.\n",
			    __func__);
			break;
		}
		/* Read TLV length */
		status = ice_read_sr_word(hw, next_tlv + 1, &tlv_len);
		if (status != ICE_SUCCESS) {
			DNPRINTF(ICE_DBG_INIT,
			    "%s: Failed to read TLV length.\n", __func__);
			break;
		}
		if (tlv_sub_module_type == module_type) {
			if (tlv_len) {
				*module_tlv = next_tlv;
				*module_tlv_len = tlv_len;
				return ICE_SUCCESS;
			}
			return ICE_ERR_INVAL_SIZE;
		}
		/* Check next TLV, i.e. current TLV pointer + length + 2 words
		 * (for current TLV's type and length)
		 */
		next_tlv = next_tlv + tlv_len + 2;
	}
	/* Module does not exist */
	return ICE_ERR_DOES_NOT_EXIST;
}

/**
 * ice_get_link_default_override
 * @ldo: pointer to the link default override struct
 * @pi: pointer to the port info struct
 *
 * Gets the link default override for a port
 */
enum ice_status
ice_get_link_default_override(struct ice_link_default_override_tlv *ldo,
			      struct ice_port_info *pi)
{
	uint16_t i, tlv, tlv_len, tlv_start, buf, offset;
	struct ice_hw *hw = pi->hw;
	enum ice_status status;

	status = ice_get_pfa_module_tlv(hw, &tlv, &tlv_len,
					ICE_SR_LINK_DEFAULT_OVERRIDE_PTR);
	if (status) {
		DNPRINTF(ICE_DBG_INIT,
		     "%s: Failed to read link override TLV.\n", __func__);
		return status;
	}

	/* Each port has its own config; calculate for our port */
	tlv_start = tlv + pi->lport * ICE_SR_PFA_LINK_OVERRIDE_WORDS +
		ICE_SR_PFA_LINK_OVERRIDE_OFFSET;

	/* link options first */
	status = ice_read_sr_word(hw, tlv_start, &buf);
	if (status) {
		DNPRINTF(ICE_DBG_INIT,
		    "%s: Failed to read override link options.\n", __func__);
		return status;
	}
	ldo->options = buf & ICE_LINK_OVERRIDE_OPT_M;
	ldo->phy_config = (buf & ICE_LINK_OVERRIDE_PHY_CFG_M) >>
		ICE_LINK_OVERRIDE_PHY_CFG_S;

	/* link PHY config */
	offset = tlv_start + ICE_SR_PFA_LINK_OVERRIDE_FEC_OFFSET;
	status = ice_read_sr_word(hw, offset, &buf);
	if (status) {
		DNPRINTF(ICE_DBG_INIT,
		    "%s: Failed to read override phy config.\n", __func__);
		return status;
	}
	ldo->fec_options = buf & ICE_LINK_OVERRIDE_FEC_OPT_M;

	/* PHY types low */
	offset = tlv_start + ICE_SR_PFA_LINK_OVERRIDE_PHY_OFFSET;
	for (i = 0; i < ICE_SR_PFA_LINK_OVERRIDE_PHY_WORDS; i++) {
		status = ice_read_sr_word(hw, (offset + i), &buf);
		if (status) {
			DNPRINTF(ICE_DBG_INIT,
			    "%s: Failed to read override link options.\n",
			    __func__);
			return status;
		}
		/* shift 16 bits at a time to fill 64 bits */
		ldo->phy_type_low |= ((uint64_t)buf << (i * 16));
	}

	/* PHY types high */
	offset = tlv_start + ICE_SR_PFA_LINK_OVERRIDE_PHY_OFFSET +
		ICE_SR_PFA_LINK_OVERRIDE_PHY_WORDS;
	for (i = 0; i < ICE_SR_PFA_LINK_OVERRIDE_PHY_WORDS; i++) {
		status = ice_read_sr_word(hw, (offset + i), &buf);
		if (status) {
			DNPRINTF(ICE_DBG_INIT,
			    "%s: Failed to read override link options.\n",
			    __func__);
			return status;
		}
		/* shift 16 bits at a time to fill 64 bits */
		ldo->phy_type_high |= ((uint64_t)buf << (i * 16));
	}

	return status;
}

/**
 * ice_fw_supports_fec_dis_auto
 * @hw: pointer to the hardware structure
 *
 * Checks if the firmware supports FEC disable in Auto FEC mode
 */
bool ice_fw_supports_fec_dis_auto(struct ice_hw *hw)
{
	return ice_is_fw_min_ver(hw, ICE_FW_VER_BRANCH_E810,
				 ICE_FW_FEC_DIS_AUTO_MAJ,
				 ICE_FW_FEC_DIS_AUTO_MIN,
				 ICE_FW_FEC_DIS_AUTO_PATCH) ||
	       ice_is_fw_min_ver(hw, ICE_FW_VER_BRANCH_E82X,
				 ICE_FW_FEC_DIS_AUTO_MAJ_E82X,
				 ICE_FW_FEC_DIS_AUTO_MIN_E82X,
				 ICE_FW_FEC_DIS_AUTO_PATCH_E82X);
}

/**
 * ice_cfg_phy_fec - Configure PHY FEC data based on FEC mode
 * @pi: port information structure
 * @cfg: PHY configuration data to set FEC mode
 * @fec: FEC mode to configure
 */
enum ice_status
ice_cfg_phy_fec(struct ice_port_info *pi, struct ice_aqc_set_phy_cfg_data *cfg,
		enum ice_fec_mode fec)
{
	struct ice_aqc_get_phy_caps_data *pcaps;
	enum ice_status status = ICE_SUCCESS;
	struct ice_hw *hw;

	if (!pi || !cfg)
		return ICE_ERR_BAD_PTR;

	hw = pi->hw;

	pcaps = (struct ice_aqc_get_phy_caps_data *)
		ice_malloc(hw, sizeof(*pcaps));
	if (!pcaps)
		return ICE_ERR_NO_MEMORY;

	status = ice_aq_get_phy_caps(pi, false,
				     (ice_fw_supports_report_dflt_cfg(hw) ?
				      ICE_AQC_REPORT_DFLT_CFG :
				      ICE_AQC_REPORT_TOPO_CAP_MEDIA), pcaps,
				      NULL);

	if (status)
		goto out;

	cfg->caps |= (pcaps->caps & ICE_AQC_PHY_EN_AUTO_FEC);
	cfg->link_fec_opt = pcaps->link_fec_options;

	switch (fec) {
	case ICE_FEC_BASER:
		/* Clear RS bits, and AND BASE-R ability
		 * bits and OR request bits.
		 */
		cfg->link_fec_opt &= ICE_AQC_PHY_FEC_10G_KR_40G_KR4_EN |
			ICE_AQC_PHY_FEC_25G_KR_CLAUSE74_EN;
		cfg->link_fec_opt |= ICE_AQC_PHY_FEC_10G_KR_40G_KR4_REQ |
			ICE_AQC_PHY_FEC_25G_KR_REQ;
		break;
	case ICE_FEC_RS:
		/* Clear BASE-R bits, and AND RS ability
		 * bits and OR request bits.
		 */
		cfg->link_fec_opt &= ICE_AQC_PHY_FEC_25G_RS_CLAUSE91_EN;
		cfg->link_fec_opt |= ICE_AQC_PHY_FEC_25G_RS_528_REQ |
			ICE_AQC_PHY_FEC_25G_RS_544_REQ;
		break;
	case ICE_FEC_NONE:
		/* Clear all FEC option bits. */
		cfg->link_fec_opt &= ~ICE_AQC_PHY_FEC_MASK;
		break;
	case ICE_FEC_DIS_AUTO:
		/* Set No FEC and auto FEC */
		if (!ice_fw_supports_fec_dis_auto(hw)) {
			status = ICE_ERR_NOT_SUPPORTED;
			goto out;
		}
		cfg->link_fec_opt |= ICE_AQC_PHY_FEC_DIS;
		/* fall-through */
	case ICE_FEC_AUTO:
		/* AND auto FEC bit, and all caps bits. */
		cfg->caps &= ICE_AQC_PHY_CAPS_MASK;
		cfg->link_fec_opt |= pcaps->link_fec_options;
		break;
	default:
		status = ICE_ERR_PARAM;
		break;
	}

	if (fec == ICE_FEC_AUTO && ice_fw_supports_link_override(pi->hw) &&
	    !ice_fw_supports_report_dflt_cfg(pi->hw)) {
		struct ice_link_default_override_tlv tlv;

		if (ice_get_link_default_override(&tlv, pi))
			goto out;

		if (!(tlv.options & ICE_LINK_OVERRIDE_STRICT_MODE) &&
		    (tlv.options & ICE_LINK_OVERRIDE_EN))
			cfg->link_fec_opt = tlv.fec_options;
	}

out:
	ice_free(hw, pcaps);

	return status;
}

/**
 * ice_apply_saved_fec_req_to_cfg -- Write saved user FEC mode to cfg data
 * @sc: device private structure
 * @cfg: new PHY config data to be modified
 *
 * Applies user setting for FEC mode to PHY config struct. It uses the data
 * from pcaps to check if the saved settings are invalid and uses the pcaps
 * data instead if they are invalid.
 */
int
ice_apply_saved_fec_req_to_cfg(struct ice_softc *sc,
			       struct ice_aqc_set_phy_cfg_data *cfg)
{
	struct ice_port_info *pi = sc->hw.port_info;
	enum ice_status status;

	cfg->caps &= ~ICE_AQC_PHY_EN_AUTO_FEC;
	status = ice_cfg_phy_fec(pi, cfg, pi->phy.curr_user_fec_req);
	if (status)
		return (EIO);

	return (0);
}

/**
 * ice_apply_saved_fc_req_to_cfg -- Write saved user flow control mode to cfg data
 * @pi: port info struct
 * @cfg: new PHY config data to be modified
 *
 * Applies user setting for flow control mode to PHY config struct. There are
 * no invalid flow control mode settings; if there are, then this function
 * treats them like "ICE_FC_NONE".
 */
void
ice_apply_saved_fc_req_to_cfg(struct ice_port_info *pi,
			      struct ice_aqc_set_phy_cfg_data *cfg)
{
	cfg->caps &= ~(ICE_AQ_PHY_ENA_TX_PAUSE_ABILITY |
		       ICE_AQ_PHY_ENA_RX_PAUSE_ABILITY);

	switch (pi->phy.curr_user_fc_req) {
	case ICE_FC_FULL:
		cfg->caps |= ICE_AQ_PHY_ENA_TX_PAUSE_ABILITY |
			     ICE_AQ_PHY_ENA_RX_PAUSE_ABILITY;
		break;
	case ICE_FC_RX_PAUSE:
		cfg->caps |= ICE_AQ_PHY_ENA_RX_PAUSE_ABILITY;
		break;
	case ICE_FC_TX_PAUSE:
		cfg->caps |= ICE_AQ_PHY_ENA_TX_PAUSE_ABILITY;
		break;
	default:
		/* ICE_FC_NONE */
		break;
	}
}

/**
 * ice_caps_to_fc_mode
 * @caps: PHY capabilities
 *
 * Convert PHY FC capabilities to ice FC mode
 */
enum ice_fc_mode
ice_caps_to_fc_mode(uint8_t caps)
{
	if (caps & ICE_AQC_PHY_EN_TX_LINK_PAUSE &&
	    caps & ICE_AQC_PHY_EN_RX_LINK_PAUSE)
		return ICE_FC_FULL;

	if (caps & ICE_AQC_PHY_EN_TX_LINK_PAUSE)
		return ICE_FC_TX_PAUSE;

	if (caps & ICE_AQC_PHY_EN_RX_LINK_PAUSE)
		return ICE_FC_RX_PAUSE;

	return ICE_FC_NONE;
}

/**
 * ice_caps_to_fec_mode
 * @caps: PHY capabilities
 * @fec_options: Link FEC options
 *
 * Convert PHY FEC capabilities to ice FEC mode
 */
enum ice_fec_mode
ice_caps_to_fec_mode(uint8_t caps, uint8_t fec_options)
{
	if (caps & ICE_AQC_PHY_EN_AUTO_FEC) {
		if (fec_options & ICE_AQC_PHY_FEC_DIS)
			return ICE_FEC_DIS_AUTO;
		else
			return ICE_FEC_AUTO;
	}

	if (fec_options & (ICE_AQC_PHY_FEC_10G_KR_40G_KR4_EN |
			   ICE_AQC_PHY_FEC_10G_KR_40G_KR4_REQ |
			   ICE_AQC_PHY_FEC_25G_KR_CLAUSE74_EN |
			   ICE_AQC_PHY_FEC_25G_KR_REQ))
		return ICE_FEC_BASER;

	if (fec_options & (ICE_AQC_PHY_FEC_25G_RS_528_REQ |
			   ICE_AQC_PHY_FEC_25G_RS_544_REQ |
			   ICE_AQC_PHY_FEC_25G_RS_CLAUSE91_EN))
		return ICE_FEC_RS;

	return ICE_FEC_NONE;
}

/**
 * ice_aq_set_phy_cfg
 * @hw: pointer to the HW struct
 * @pi: port info structure of the interested logical port
 * @cfg: structure with PHY configuration data to be set
 * @cd: pointer to command details structure or NULL
 *
 * Set the various PHY configuration parameters supported on the Port.
 * One or more of the Set PHY config parameters may be ignored in an MFP
 * mode as the PF may not have the privilege to set some of the PHY Config
 * parameters. This status will be indicated by the command response (0x0601).
 */
enum ice_status
ice_aq_set_phy_cfg(struct ice_hw *hw, struct ice_port_info *pi,
		   struct ice_aqc_set_phy_cfg_data *cfg, struct ice_sq_cd *cd)
{
	struct ice_aq_desc desc;
	enum ice_status status;

	if (!cfg)
		return ICE_ERR_PARAM;

	/* Ensure that only valid bits of cfg->caps can be turned on. */
	if (cfg->caps & ~ICE_AQ_PHY_ENA_VALID_MASK) {
		DNPRINTF(ICE_DBG_PHY, "%s: Invalid bit is set in "
		    "ice_aqc_set_phy_cfg_data->caps : 0x%x\n",
		    __func__, cfg->caps);

		cfg->caps &= ICE_AQ_PHY_ENA_VALID_MASK;
	}

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_set_phy_cfg);
	desc.params.set_phy.lport_num = pi->lport;
	desc.flags |= htole16(ICE_AQ_FLAG_RD);

	DNPRINTF(ICE_DBG_LINK, "set phy cfg\n");
	DNPRINTF(ICE_DBG_LINK, "	phy_type_low = 0x%llx\n",
	      (unsigned long long)le64toh(cfg->phy_type_low));
	DNPRINTF(ICE_DBG_LINK, "	phy_type_high = 0x%llx\n",
	      (unsigned long long)le64toh(cfg->phy_type_high));
	DNPRINTF(ICE_DBG_LINK, "	caps = 0x%x\n", cfg->caps);
	DNPRINTF(ICE_DBG_LINK, "	low_power_ctrl_an = 0x%x\n",
	      cfg->low_power_ctrl_an);
	DNPRINTF(ICE_DBG_LINK, "	eee_cap = 0x%x\n", cfg->eee_cap);
	DNPRINTF(ICE_DBG_LINK, "	eeer_value = 0x%x\n", cfg->eeer_value);
	DNPRINTF(ICE_DBG_LINK, "	link_fec_opt = 0x%x\n",
		  cfg->link_fec_opt);

	status = ice_aq_send_cmd(hw, &desc, cfg, sizeof(*cfg), cd);

	if (hw->adminq.sq_last_status == ICE_AQ_RC_EMODE)
		status = ICE_SUCCESS;

	if (!status)
		pi->phy.curr_user_phy_cfg = *cfg;

	return status;
}

/**
 * ice_apply_saved_phy_cfg -- Re-apply user PHY config settings
 * @sc: device private structure
 * @settings: which settings to apply
 *
 * Applies user settings for advertised speeds, FEC mode, and flow
 * control mode to a PHY config struct; it uses the data from pcaps
 * to check if the saved settings are invalid and uses the pcaps
 * data instead if they are invalid.
 *
 * For things like sysctls where only one setting needs to be
 * updated, the bitmap allows the caller to specify which setting
 * to update.
 */
int
ice_apply_saved_phy_cfg(struct ice_softc *sc, uint8_t settings)
{
	struct ice_aqc_set_phy_cfg_data cfg = { 0 };
	struct ice_port_info *pi = sc->hw.port_info;
	struct ice_aqc_get_phy_caps_data pcaps = { 0 };
	struct ice_hw *hw = &sc->hw;
	uint64_t phy_low, phy_high;
	enum ice_status status;
	enum ice_fec_mode dflt_fec_mode;
	uint16_t dflt_user_speed;

	if (!settings || settings > ICE_APPLY_LS_FEC_FC) {
		DNPRINTF(ICE_DBG_LINK, "%s: Settings out-of-bounds: %u\n",
		    __func__, settings);
		return EINVAL;
	}

	status = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_ACTIVE_CFG,
				     &pcaps, NULL);
	if (status != ICE_SUCCESS) {
		printf("%s: ice_aq_get_phy_caps (ACTIVE) failed; "
		    "status %s, aq_err %s\n",
		    sc->sc_dev.dv_xname, ice_status_str(status),
		    ice_aq_str(hw->adminq.sq_last_status));
		return (EIO);
	}

	phy_low = le64toh(pcaps.phy_type_low);
	phy_high = le64toh(pcaps.phy_type_high);

	/* Save off initial config parameters */
	dflt_user_speed = ice_aq_phy_types_to_link_speeds(phy_low, phy_high);
	dflt_fec_mode = ice_caps_to_fec_mode(pcaps.caps,
	    pcaps.link_fec_options);

	/* Setup new PHY config */
	ice_copy_phy_caps_to_cfg(pi, &pcaps, &cfg);

	/* On error, restore active configuration values */
	if ((settings & ICE_APPLY_LS) &&
	    ice_apply_saved_phy_req_to_cfg(sc, &cfg)) {
		pi->phy.curr_user_speed_req = dflt_user_speed;
		cfg.phy_type_low = pcaps.phy_type_low;
		cfg.phy_type_high = pcaps.phy_type_high;
	}
	if ((settings & ICE_APPLY_FEC) &&
	    ice_apply_saved_fec_req_to_cfg(sc, &cfg)) {
		pi->phy.curr_user_fec_req = dflt_fec_mode;
	}
	if (settings & ICE_APPLY_FC) {
		/* No real error indicators for this process,
		 * so we'll just have to assume it works. */
		ice_apply_saved_fc_req_to_cfg(pi, &cfg);
	}

	/* Enable link and re-negotiate it */
	cfg.caps |= ICE_AQ_PHY_ENA_AUTO_LINK_UPDT | ICE_AQ_PHY_ENA_LINK;

	status = ice_aq_set_phy_cfg(hw, pi, &cfg, NULL);
	if (status != ICE_SUCCESS) {
		/* Don't indicate failure if there's no media in the port.
		 * The settings have been saved and will apply when media
		 * is inserted.
		 */
		if ((status == ICE_ERR_AQ_ERROR) &&
		    (hw->adminq.sq_last_status == ICE_AQ_RC_EBUSY)) {
			DPRINTF("%s: Setting will be applied when media is "
			    "inserted\n", __func__);
			return (0);
		} else {
			printf("%s: ice_aq_set_phy_cfg failed; status %s, "
			    "aq_err %s\n", sc->sc_dev.dv_xname,
			    ice_status_str(status),
			    ice_aq_str(hw->adminq.sq_last_status));
			return (EIO);
		}
	}

	return (0);
}

/**
 * ice_aq_set_link_restart_an
 * @pi: pointer to the port information structure
 * @ena_link: if true: enable link, if false: disable link
 * @cd: pointer to command details structure or NULL
 *
 * Sets up the link and restarts the Auto-Negotiation over the link.
 */
enum ice_status
ice_aq_set_link_restart_an(struct ice_port_info *pi, bool ena_link,
			   struct ice_sq_cd *cd)
{
	enum ice_status status = ICE_ERR_AQ_ERROR;
	struct ice_aqc_restart_an *cmd;
	struct ice_aq_desc desc;

	cmd = &desc.params.restart_an;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_restart_an);

	cmd->cmd_flags = ICE_AQC_RESTART_AN_LINK_RESTART;
	cmd->lport_num = pi->lport;
	if (ena_link)
		cmd->cmd_flags |= ICE_AQC_RESTART_AN_LINK_ENABLE;
	else
		cmd->cmd_flags &= ~ICE_AQC_RESTART_AN_LINK_ENABLE;

	status = ice_aq_send_cmd(pi->hw, &desc, NULL, 0, cd);
	if (status)
		return status;

	if (ena_link)
		pi->phy.curr_user_phy_cfg.caps |= ICE_AQC_PHY_EN_LINK;
	else
		pi->phy.curr_user_phy_cfg.caps &= ~ICE_AQC_PHY_EN_LINK;

	return ICE_SUCCESS;
}

/**
 * ice_set_link -- Set up/down link on phy
 * @sc: device private structure
 * @enabled: link status to set up
 *
 * This should be called when change of link status is needed.
 */
void
ice_set_link(struct ice_softc *sc, bool enabled)
{
	struct ice_hw *hw = &sc->hw;
	enum ice_status status;

	if (ice_driver_is_detaching(sc))
		return;

	if (ice_test_state(&sc->state, ICE_STATE_NO_MEDIA))
		return;

	if (enabled)
		ice_apply_saved_phy_cfg(sc, ICE_APPLY_LS_FEC_FC);
	else {
		status = ice_aq_set_link_restart_an(hw->port_info, false, NULL);
		if (status != ICE_SUCCESS) {
			if (hw->adminq.sq_last_status == ICE_AQ_RC_EMODE)
				printf("%s: Link control not enabled in "
				    "current device mode\n",
				    sc->sc_dev.dv_xname);
			else
				printf("%s: could not restart link: status %s, "
				    "aq_err %s\n", sc->sc_dev.dv_xname,
				    ice_status_str(status),
				    ice_aq_str(hw->adminq.sq_last_status));
		} else
			sc->link_up = false;
	}
}

int
ice_up(struct ice_softc *sc)
{
	struct ifnet *ifp = &sc->sc_ac.ac_if;
	struct ice_vsi *vsi = &sc->pf_vsi;
	struct ice_rx_queue *rxq;
	struct ice_tx_queue *txq;
	struct ifiqueue *ifiq;
	struct ice_intr_vector *iv;
	int i, err;

	ice_update_laa_mac(sc);

	/* Initialize software Tx tracking values */
	ice_init_tx_tracking(&sc->pf_vsi);

	for (i = 0, rxq = vsi->rx_queues; i < vsi->num_rx_queues; i++, rxq++) {
		ice_rxfill(sc, rxq);

		/* wire everything together */
		iv = &sc->sc_vectors[i];
		iv->iv_rxq = rxq;
		rxq->irqv = iv;

		ifiq = ifp->if_iqs[i];
		ifiq->ifiq_softc = rxq;
		rxq->rxq_ifiq = ifiq;
	}

	for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) {
		/* wire everything together */
		iv = &sc->sc_vectors[i];
		iv->iv_txq = txq;
		txq->irqv = iv;
	}

	err = ice_cfg_vsi_for_tx(&sc->pf_vsi);
	if (err) {
		printf("%s: Unable to configure the main VSI for Tx: err %d\n",
		    sc->sc_dev.dv_xname, err);
		return err;
	}

	err = ice_cfg_vsi_for_rx(&sc->pf_vsi);
	if (err) {
		printf("%s: Unable to configure the main VSI for Rx: err %d\n",
		    sc->sc_dev.dv_xname, err);
		goto err_cleanup_tx;
	}

	err = ice_control_all_rx_queues(&sc->pf_vsi, true);
	if (err) {
		printf("%s: Could not enable Rx rings: err %d\n",
		    sc->sc_dev.dv_xname, err);
		goto err_cleanup_tx;
	}

	err = ice_cfg_pf_default_mac_filters(sc);
	if (err) {
		printf("%s: Unable to configure default MAC filters: %d\n",
		    sc->sc_dev.dv_xname, err);
		goto err_stop_rx;
	}

	ice_configure_all_rxq_interrupts(&sc->pf_vsi);
	ice_configure_rx_itr(&sc->pf_vsi);

	/* Configure promiscuous mode */
	ice_if_promisc_set(sc);

	if (!ice_testandclear_state(&sc->state, ICE_STATE_FIRST_INIT_LINK) &&
	    (!sc->link_up && ((ifp->if_flags & IFF_UP) ||
	    ice_test_state(&sc->state, ICE_STATE_LINK_ACTIVE_ON_DOWN))))
		ice_set_link(sc, true);

	/* Enable Rx queue interrupts */
	for (i = 0; i < vsi->num_rx_queues; i++) {
		int v = vsi->rx_queues[i].irqv->iv_qid + 1;
		ice_enable_intr(&sc->hw, v);
	}

	timeout_add_nsec(&sc->sc_admin_timer, SEC_TO_NSEC(1));

	ifp->if_flags |= IFF_RUNNING;

	ice_set_state(&sc->state, ICE_STATE_DRIVER_INITIALIZED);
	return 0;

err_stop_rx:
	ice_control_all_rx_queues(&sc->pf_vsi, false);
err_cleanup_tx:
	ice_vsi_disable_tx(&sc->pf_vsi);
	return err;
}

/**
 * ice_find_ucast_rule_entry - Search for a unicast MAC filter rule entry
 * @list_head: head of rule list
 * @f_info: rule information
 *
 * Helper function to search for a unicast rule entry - this is to be used
 * to remove unicast MAC filter that is not shared with other VSIs on the
 * PF switch.
 *
 * Returns pointer to entry storing the rule if found
 */
struct ice_fltr_mgmt_list_entry *
ice_find_ucast_rule_entry(struct ice_fltr_mgmt_list_head *list_head,
			  struct ice_fltr_info *f_info)
{
	struct ice_fltr_mgmt_list_entry *list_itr;

	TAILQ_FOREACH(list_itr, list_head, list_entry) {
		if (!memcmp(&f_info->l_data, &list_itr->fltr_info.l_data,
			    sizeof(f_info->l_data)) &&
		    f_info->fwd_id.hw_vsi_id ==
		    list_itr->fltr_info.fwd_id.hw_vsi_id &&
		    f_info->flag == list_itr->fltr_info.flag)
			return list_itr;
	}
	return NULL;
}

/**
 * ice_remove_mac_rule - remove a MAC based filter rule
 * @hw: pointer to the hardware structure
 * @m_list: list of MAC addresses and forwarding information
 * @recp_list: list from which function remove MAC address
 *
 * This function removes either a MAC filter rule or a specific VSI from a
 * VSI list for a multicast MAC address.
 *
 * Returns ICE_ERR_DOES_NOT_EXIST if a given entry was not added by
 * ice_add_mac. Caller should be aware that this call will only work if all
 * the entries passed into m_list were added previously. It will not attempt to
 * do a partial remove of entries that were found.
 */
enum ice_status
ice_remove_mac_rule(struct ice_hw *hw, struct ice_fltr_list_head *m_list,
		    struct ice_sw_recipe *recp_list)
{
	struct ice_fltr_list_entry *list_itr, *tmp;
	struct ice_lock *rule_lock; /* Lock to protect filter rule list */

	if (!m_list)
		return ICE_ERR_PARAM;

	rule_lock = &recp_list->filt_rule_lock;
	TAILQ_FOREACH_SAFE(list_itr, m_list, list_entry, tmp) {
		enum ice_sw_lkup_type l_type = list_itr->fltr_info.lkup_type;
		uint8_t *add = &list_itr->fltr_info.l_data.mac.mac_addr[0];
		uint16_t vsi_handle;

		if (l_type != ICE_SW_LKUP_MAC)
			return ICE_ERR_PARAM;

		vsi_handle = list_itr->fltr_info.vsi_handle;
		if (!ice_is_vsi_valid(hw, vsi_handle))
			return ICE_ERR_PARAM;

		list_itr->fltr_info.fwd_id.hw_vsi_id =
		    hw->vsi_ctx[vsi_handle]->vsi_num;
		if (!ETHER_IS_MULTICAST(add) && !hw->umac_shared) {
			/* Don't remove the unicast address that belongs to
			 * another VSI on the switch, since it is not being
			 * shared...
			 */
#if 0
			ice_acquire_lock(rule_lock);
#endif
			if (!ice_find_ucast_rule_entry(&recp_list->filt_rules,
						       &list_itr->fltr_info)) {
#if 0
				ice_release_lock(rule_lock);
#endif
				return ICE_ERR_DOES_NOT_EXIST;
			}
#if 0
			ice_release_lock(rule_lock);
#endif
		}
		list_itr->status = ice_remove_rule_internal(hw, recp_list,
							    list_itr);
		if (list_itr->status)
			return list_itr->status;
	}
	return ICE_SUCCESS;
}

/**
 * ice_remove_mac - remove a MAC address based filter rule
 * @hw: pointer to the hardware structure
 * @m_list: list of MAC addresses and forwarding information
 *
 */
enum ice_status
ice_remove_mac(struct ice_hw *hw, struct ice_fltr_list_head *m_list)
{
	struct ice_sw_recipe *recp_list;

	recp_list = &hw->switch_info->recp_list[ICE_SW_LKUP_MAC];
	return ice_remove_mac_rule(hw, m_list, recp_list);
}

/**
 * ice_remove_vsi_mac_filter - Remove a MAC address filter for a VSI
 * @vsi: the VSI to add the filter for
 * @addr: MAC address to remove a filter for
 *
 * Remove a MAC address filter from a given VSI. This is a wrapper around
 * ice_remove_mac to simplify the interface. First, it only accepts a single
 * address, so we don't have to mess around with the list setup in other
 * functions. Second, it ignores the ICE_ERR_DOES_NOT_EXIST error, so that
 * callers don't need to worry about attempting to remove filters which
 * haven't yet been added.
 */
int
ice_remove_vsi_mac_filter(struct ice_vsi *vsi, uint8_t *addr)
{
	struct ice_softc *sc = vsi->sc;
	struct ice_fltr_list_head mac_addr_list;
	struct ice_hw *hw = &sc->hw;
	enum ice_status status;
	int err = 0;

	TAILQ_INIT(&mac_addr_list);

	err = ice_add_mac_to_list(vsi, &mac_addr_list, addr, ICE_FWD_TO_VSI);
	if (err)
		goto free_mac_list;

	status = ice_remove_mac(hw, &mac_addr_list);
	if (status && status != ICE_ERR_DOES_NOT_EXIST) {
		DPRINTF("%s: failed to remove a filter for MAC %s, "
		    "err %s aq_err %s\n", __func__,
		    ether_sprintf(addr), ice_status_str(status),
		    ice_aq_str(hw->adminq.sq_last_status));
		err = EIO;
	}

free_mac_list:
	ice_free_fltr_list(&mac_addr_list);
	return err;
}

/**
 * ice_rm_pf_default_mac_filters - Remove default unicast and broadcast addrs
 * @sc: device softc structure
 *
 * Remove the default unicast and broadcast filters from the PF VSI.
 */
int
ice_rm_pf_default_mac_filters(struct ice_softc *sc)
{
	struct ice_vsi *vsi = &sc->pf_vsi;
	struct ice_hw *hw = &sc->hw;
	int err;

	/* Remove the LAN MAC address */
	err = ice_remove_vsi_mac_filter(vsi, hw->port_info->mac.lan_addr);
	if (err)
		return err;

	/* Remove the broadcast address */
	err = ice_remove_vsi_mac_filter(vsi, etherbroadcastaddr);
	if (err)
		return (EIO);

	return (0);
}

/**
 * ice_flush_rxq_interrupts - Unconfigure Hw Rx queues MSI-X interrupt cause
 * @vsi: the VSI to configure
 *
 * Unset the CAUSE_ENA flag of the TQCTL register for each queue, then trigger
 * a software interrupt on that cause. This is required as part of the Rx
 * queue disable logic to dissociate the Rx queue from the interrupt.
 *
 * This function must be called prior to disabling Rx queues with
 * ice_control_all_rx_queues, otherwise the Rx queue may not be disabled
 * properly.
 */
void
ice_flush_rxq_interrupts(struct ice_vsi *vsi)
{
	struct ice_softc *sc = vsi->sc;
	struct ice_hw *hw = &sc->hw;
	int i;

	for (i = 0; i < vsi->num_rx_queues; i++) {
		struct ice_rx_queue *rxq = &vsi->rx_queues[i];
		uint32_t reg, val;
		int v = rxq->irqv->iv_qid + 1;
		int tries = 0;

		/* Clear the CAUSE_ENA flag */
		reg = vsi->rx_qmap[rxq->me];
		val = ICE_READ(hw, QINT_RQCTL(reg));
		val &= ~QINT_RQCTL_CAUSE_ENA_M;
		ICE_WRITE(hw, QINT_RQCTL(reg), val);

		ice_flush(hw);

		/* Trigger a software interrupt to complete interrupt
		 * dissociation.
		 */
		sc->sw_intr[v] = -1;
		ICE_WRITE(hw, GLINT_DYN_CTL(v),
		     GLINT_DYN_CTL_SWINT_TRIG_M | GLINT_DYN_CTL_INTENA_MSK_M);
		do {
			int ret;

			/* Sleep to allow interrupt processing to occur. */
			ret = tsleep_nsec(&sc->sw_intr[v], 0, "iceswi",
			    USEC_TO_NSEC(1));
			if (ret == 0 && sc->sw_intr[v] == 1) {
				sc->sw_intr[v] = 0;
				break;
			}
			tries++;
		} while (tries < 10);
		if (tries == 10)
			DPRINTF("%s: missed software interrupt\n", __func__);
	}
}

/**
 * ice_flush_txq_interrupts - Unconfigure Hw Tx queues MSI-X interrupt cause
 * @vsi: the VSI to configure
 *
 * Unset the CAUSE_ENA flag of the TQCTL register for each queue, then trigger
 * a software interrupt on that cause. This is required as part of the Tx
 * queue disable logic to dissociate the Tx queue from the interrupt.
 *
 * This function must be called prior to ice_vsi_disable_tx, otherwise
 * the Tx queue disable may not complete properly.
 */
void
ice_flush_txq_interrupts(struct ice_vsi *vsi)
{
	struct ice_softc *sc = vsi->sc;
	struct ice_hw *hw = &sc->hw;
	int i;

	for (i = 0; i < vsi->num_tx_queues; i++) {
		struct ice_tx_queue *txq = &vsi->tx_queues[i];
		uint32_t reg, val;
		int v = txq->irqv->iv_qid + 1;
		int tries = 0;

		/* Clear the CAUSE_ENA flag */
		reg = vsi->tx_qmap[txq->me];
		val = ICE_READ(hw, QINT_TQCTL(reg));
		val &= ~QINT_TQCTL_CAUSE_ENA_M;
		ICE_WRITE(hw, QINT_TQCTL(reg), val);

		ice_flush(hw);

		/* Trigger a software interrupt to complete interrupt
		 * dissociation.
		 */
		sc->sw_intr[v] = -1;
		ICE_WRITE(hw, GLINT_DYN_CTL(v),
		     GLINT_DYN_CTL_SWINT_TRIG_M | GLINT_DYN_CTL_INTENA_MSK_M);
		do {
			int ret;

			/* Sleep to allow interrupt processing to occur. */
			ret = tsleep_nsec(&sc->sw_intr[v], 0, "iceswi",
			    USEC_TO_NSEC(1));
			if (ret == 0 && sc->sw_intr[v] == 1) {
				sc->sw_intr[v] = 0;
				break;
			}
			tries++;
		} while (tries < 10);
		if (tries == 10)
			DPRINTF("%s: missed software interrupt\n", __func__);
	}
}

int
ice_down(struct ice_softc *sc)
{
	struct ifnet *ifp = &sc->sc_ac.ac_if;

	timeout_del(&sc->sc_admin_timer);
	ifp->if_flags &= ~IFF_RUNNING;
#if 0
	ASSERT_CTX_LOCKED(sc);
#endif
	if (!ice_testandclear_state(&sc->state, ICE_STATE_DRIVER_INITIALIZED))
		return 0;

	if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED)) {
		printf("%s: request to stop interface cannot be completed "
		    "as the device failed to reset\n", sc->sc_dev.dv_xname);
		return ENODEV;
	}

	if (ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) {
		DPRINTF("%s: request to stop interface while device is "
		    "prepared for impending reset\n", __func__);
		return EBUSY;
	}
#if 0
	ice_rdma_pf_stop(sc);
#endif
	/* Remove the MAC filters, stop Tx, and stop Rx. We don't check the
	 * return of these functions because there's nothing we can really do
	 * if they fail, and the functions already print error messages.
	 * Just try to shut down as much as we can.
	 */
	ice_rm_pf_default_mac_filters(sc);

	/* Dissociate the Tx and Rx queues from the interrupts */
	ice_flush_txq_interrupts(&sc->pf_vsi);
	ice_flush_rxq_interrupts(&sc->pf_vsi);

	/* Disable the Tx and Rx queues */
	ice_vsi_disable_tx(&sc->pf_vsi);
	ice_control_all_rx_queues(&sc->pf_vsi, false);

	if (!ice_test_state(&sc->state, ICE_STATE_LINK_ACTIVE_ON_DOWN) &&
		 !(ifp->if_flags & IFF_UP) && sc->link_up)
		ice_set_link(sc, false);
#if 0
	if (sc->mirr_if && ice_test_state(&mif->state, ICE_STATE_SUBIF_NEEDS_REINIT)) {
		ice_subif_if_stop(sc->mirr_if->subctx);
		device_printf(sc->dev, "The subinterface also comes down and up after reset\n");
	}
#endif

	return 0;
}

int
ice_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
{
	struct ice_softc *sc = ifp->if_softc;
	struct ifreq *ifr = (struct ifreq *)data;
	int s, error = 0;

	s = splnet();

	switch (cmd) {
	case SIOCSIFADDR:
		ifp->if_flags |= IFF_UP;
		/* FALLTHROUGH */

	case SIOCSIFFLAGS:
		if (ISSET(ifp->if_flags, IFF_UP)) {
			if (ISSET(ifp->if_flags, IFF_RUNNING))
				error = ENETRESET;
			else
				error = ice_up(sc);
		} else {
			if (ISSET(ifp->if_flags, IFF_RUNNING))
				error = ice_down(sc);
		}
		break;
	case SIOCSIFMEDIA:
	case SIOCGIFMEDIA:
		error = ifmedia_ioctl(ifp, ifr, &sc->media, cmd);
		break;
	default:
		error = ether_ioctl(ifp, &sc->sc_ac, cmd, data);
		break;
	}

	if (error == ENETRESET) {
		error = 0;
		if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) ==
		    (IFF_UP | IFF_RUNNING)) {
			ice_down(sc);
			error = ice_up(sc);
		}
	}

	splx(s);
	return error;
}

void
ice_start(struct ifqueue *ifq)
{
	printf("%s\n", __func__);
}

void
ice_watchdog(struct ifnet * ifp)
{
	printf("%s\n", __func__);
}

int
ice_media_change(struct ifnet *ifp)
{
	printf("%s\n", __func__);
	return ENXIO;
}

/**
 * ice_get_phy_type_low - Get media associated with phy_type_low
 * @phy_type_low: the low 64bits of phy_type from the AdminQ
 *
 * Given the lower 64bits of the phy_type from the hardware, return the
 * ifm_active bit associated. Return IFM_INST_ANY when phy_type_low is unknown.
 * Note that only one of ice_get_phy_type_low or ice_get_phy_type_high should
 * be called. If phy_type_low is zero, call ice_phy_type_high.
 */
uint64_t
ice_get_phy_type_low(uint64_t phy_type_low)
{
	switch (phy_type_low) {
	case ICE_PHY_TYPE_LOW_100BASE_TX:
		return IFM_100_TX;
#if 0
	case ICE_PHY_TYPE_LOW_100M_SGMII:
		return IFM_100_SGMII;
#endif
	case ICE_PHY_TYPE_LOW_1000BASE_T:
		return IFM_1000_T;
	case ICE_PHY_TYPE_LOW_1000BASE_SX:
		return IFM_1000_SX;
	case ICE_PHY_TYPE_LOW_1000BASE_LX:
		return IFM_1000_LX;
	case ICE_PHY_TYPE_LOW_1000BASE_KX:
		return IFM_1000_KX;
#if 0
	case ICE_PHY_TYPE_LOW_1G_SGMII:
		return IFM_1000_SGMII;
#endif
	case ICE_PHY_TYPE_LOW_2500BASE_T:
		return IFM_2500_T;
#if 0
	case ICE_PHY_TYPE_LOW_2500BASE_X:
		return IFM_2500_X;
#endif
	case ICE_PHY_TYPE_LOW_2500BASE_KX:
		return IFM_2500_KX;
	case ICE_PHY_TYPE_LOW_5GBASE_T:
		return IFM_5000_T;
#if 0
	case ICE_PHY_TYPE_LOW_5GBASE_KR:
		return IFM_5000_KR;
#endif
	case ICE_PHY_TYPE_LOW_10GBASE_T:
		return IFM_10G_T;
	case ICE_PHY_TYPE_LOW_10G_SFI_DA:
		return IFM_10G_SFP_CU;
	case ICE_PHY_TYPE_LOW_10GBASE_SR:
		return IFM_10G_SR;
	case ICE_PHY_TYPE_LOW_10GBASE_LR:
		return IFM_10G_LR;
	case ICE_PHY_TYPE_LOW_10GBASE_KR_CR1:
		return IFM_10G_KR;
	case ICE_PHY_TYPE_LOW_10G_SFI_AOC_ACC:
		return IFM_10G_AOC;
	case ICE_PHY_TYPE_LOW_10G_SFI_C2C:
		return IFM_10G_SFI;
#if 0
	case ICE_PHY_TYPE_LOW_25GBASE_T:
		return IFM_25G_T;
#endif
	case ICE_PHY_TYPE_LOW_25GBASE_CR:
		return IFM_25G_CR;
#if 0
	case ICE_PHY_TYPE_LOW_25GBASE_CR_S:
		return IFM_25G_CR_S;
	case ICE_PHY_TYPE_LOW_25GBASE_CR1:
		return IFM_25G_CR1;
#endif
	case ICE_PHY_TYPE_LOW_25GBASE_SR:
		return IFM_25G_SR;
	case ICE_PHY_TYPE_LOW_25GBASE_LR:
		return IFM_25G_LR;
	case ICE_PHY_TYPE_LOW_25GBASE_KR:
		return IFM_25G_KR;
#if 0
	case ICE_PHY_TYPE_LOW_25GBASE_KR_S:
		return IFM_25G_KR_S;
	case ICE_PHY_TYPE_LOW_25GBASE_KR1:
		return IFM_25G_KR1;
#endif
	case ICE_PHY_TYPE_LOW_25G_AUI_AOC_ACC:
		return IFM_25G_AOC;
#if 0
	case ICE_PHY_TYPE_LOW_25G_AUI_C2C:
		return IFM_25G_AUI;
#endif
	case ICE_PHY_TYPE_LOW_40GBASE_CR4:
		return IFM_40G_CR4;
	case ICE_PHY_TYPE_LOW_40GBASE_SR4:
		return IFM_40G_SR4;
	case ICE_PHY_TYPE_LOW_40GBASE_LR4:
		return IFM_40G_LR4;
	case ICE_PHY_TYPE_LOW_40GBASE_KR4:
		return IFM_40G_KR4;
#if 0
	case ICE_PHY_TYPE_LOW_40G_XLAUI_AOC_ACC:
		return IFM_40G_XLAUI_AC;
	case ICE_PHY_TYPE_LOW_40G_XLAUI:
		return IFM_40G_XLAUI;
#endif
	case ICE_PHY_TYPE_LOW_50GBASE_CR2:
		return IFM_50G_CR2;
#if 0
	case ICE_PHY_TYPE_LOW_50GBASE_SR2:
		return IFM_50G_SR2;
	case ICE_PHY_TYPE_LOW_50GBASE_LR2:
		return IFM_50G_LR2;
#endif
	case ICE_PHY_TYPE_LOW_50GBASE_KR2:
		return IFM_50G_KR2;
#if 0
	case ICE_PHY_TYPE_LOW_50G_LAUI2_AOC_ACC:
		return IFM_50G_LAUI2_AC;
	case ICE_PHY_TYPE_LOW_50G_LAUI2:
		return IFM_50G_LAUI2;
	case ICE_PHY_TYPE_LOW_50G_AUI2_AOC_ACC:
		return IFM_50G_AUI2_AC;
	case ICE_PHY_TYPE_LOW_50G_AUI2:
		return IFM_50G_AUI2;
	case ICE_PHY_TYPE_LOW_50GBASE_CP:
		return IFM_50G_CP;
	case ICE_PHY_TYPE_LOW_50GBASE_SR:
		return IFM_50G_SR;
	case ICE_PHY_TYPE_LOW_50GBASE_FR:
		return IFM_50G_FR;
	case ICE_PHY_TYPE_LOW_50GBASE_LR:
		return IFM_50G_LR;
	case ICE_PHY_TYPE_LOW_50GBASE_KR_PAM4:
		return IFM_50G_KR_PAM4;
	case ICE_PHY_TYPE_LOW_50G_AUI1_AOC_ACC:
		return IFM_50G_AUI1_AC;
	case ICE_PHY_TYPE_LOW_50G_AUI1:
		return IFM_50G_AUI1;
#endif
	case ICE_PHY_TYPE_LOW_100GBASE_CR4:
		return IFM_100G_CR4;
	case ICE_PHY_TYPE_LOW_100GBASE_SR4:
		return IFM_100G_SR4;
	case ICE_PHY_TYPE_LOW_100GBASE_LR4:
		return IFM_100G_LR4;
	case ICE_PHY_TYPE_LOW_100GBASE_KR4:
		return IFM_100G_KR4;
#if 0
	case ICE_PHY_TYPE_LOW_100G_CAUI4_AOC_ACC:
		return IFM_100G_CAUI4_AC;
	case ICE_PHY_TYPE_LOW_100G_CAUI4:
		return IFM_100G_CAUI4;
	case ICE_PHY_TYPE_LOW_100G_AUI4_AOC_ACC:
		return IFM_100G_AUI4_AC;
	case ICE_PHY_TYPE_LOW_100G_AUI4:
		return IFM_100G_AUI4;
	case ICE_PHY_TYPE_LOW_100GBASE_CR_PAM4:
		return IFM_100G_CR_PAM4;
	case ICE_PHY_TYPE_LOW_100GBASE_KR_PAM4:
		return IFM_100G_KR_PAM4;
	case ICE_PHY_TYPE_LOW_100GBASE_CP2:
		return IFM_100G_CP2;
	case ICE_PHY_TYPE_LOW_100GBASE_SR2:
		return IFM_100G_SR2;
	case ICE_PHY_TYPE_LOW_100GBASE_DR:
		return IFM_100G_DR;
#endif
	default:
		return IFM_INST_ANY;
	}
}

/**
 * ice_get_phy_type_high - Get media associated with phy_type_high
 * @phy_type_high: the upper 64bits of phy_type from the AdminQ
 *
 * Given the upper 64bits of the phy_type from the hardware, return the
 * ifm_active bit associated. Return IFM_INST_ANY on an unknown value. Note
 * that only one of ice_get_phy_type_low or ice_get_phy_type_high should be
 * called. If phy_type_high is zero, call ice_get_phy_type_low.
 */
uint64_t
ice_get_phy_type_high(uint64_t phy_type_high)
{
	switch (phy_type_high) {
#if 0
	case ICE_PHY_TYPE_HIGH_100GBASE_KR2_PAM4:
		return IFM_100G_KR2_PAM4;
	case ICE_PHY_TYPE_HIGH_100G_CAUI2_AOC_ACC:
		return IFM_100G_CAUI2_AC;
	case ICE_PHY_TYPE_HIGH_100G_CAUI2:
		return IFM_100G_CAUI2;
	case ICE_PHY_TYPE_HIGH_100G_AUI2_AOC_ACC:
		return IFM_100G_AUI2_AC;
	case ICE_PHY_TYPE_HIGH_100G_AUI2:
		return IFM_100G_AUI2;
#endif
	default:
		return IFM_INST_ANY;
	}
}

void
ice_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
{
	struct ice_softc *sc = ifp->if_softc;
	struct ice_link_status *li = &sc->hw.port_info->phy.link_info;

	ifmr->ifm_status = IFM_AVALID;
	ifmr->ifm_active = IFM_ETHER;

	/* Never report link up or media types when in recovery mode */
	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
		return;

	if (!sc->link_up)
		return;

	ifmr->ifm_status |= IFM_ACTIVE;
	ifmr->ifm_active |= IFM_FDX;

	if (li->phy_type_low)
		ifmr->ifm_active |= ice_get_phy_type_low(li->phy_type_low);
	else if (li->phy_type_high)
		ifmr->ifm_active |= ice_get_phy_type_high(li->phy_type_high);

	/* Report flow control status as well */
	if (li->an_info & ICE_AQ_LINK_PAUSE_TX)
		ifmr->ifm_active |= IFM_ETH_TXPAUSE;
	if (li->an_info & ICE_AQ_LINK_PAUSE_RX)
		ifmr->ifm_active |= IFM_ETH_RXPAUSE;
}

/**
 * ice_add_media_types - Add supported media types to the media structure
 * @sc: ice private softc structure
 * @media: ifmedia structure to setup
 *
 * Looks up the supported phy types, and initializes the various media types
 * available.
 *
 * @pre this function must be protected from being called while another thread
 * is accessing the ifmedia types.
 */
enum ice_status
ice_add_media_types(struct ice_softc *sc, struct ifmedia *media)
{
	struct ice_aqc_get_phy_caps_data pcaps = { 0 };
	struct ice_port_info *pi = sc->hw.port_info;
	enum ice_status status;
	uint64_t phy_low, phy_high;
	int bit;
#if 0
	ASSERT_CFG_LOCKED(sc);
#endif
	/* the maximum possible media type index is 511. We probably don't
	 * need most of this space, but this ensures future compatibility when
	 * additional media types are used.
	 */
	ice_declare_bitmap(already_added, 511);

	/* Remove all previous media types */
	ifmedia_delete_instance(media, IFM_INST_ANY);

	status = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_ACTIVE_CFG,
				     &pcaps, NULL);
	if (status != ICE_SUCCESS) {
		printf("%s: ice_aq_get_phy_caps (ACTIVE) failed; status %s, aq_err %s\n",
		    sc->sc_dev.dv_xname, ice_status_str(status),
		    ice_aq_str(sc->hw.adminq.sq_last_status));
		return (status);
	}

	/* make sure the added bitmap is zero'd */
	memset(already_added, 0, sizeof(already_added));

	phy_low = le64toh(pcaps.phy_type_low);
	for (bit = 0; bit < 64; bit++) {
		uint64_t type = BIT_ULL(bit);
		uint64_t ostype;

		if ((phy_low & type) == 0)
			continue;

		/* get the OS media type */
		ostype = ice_get_phy_type_low(type);
	
		/* don't bother adding the unknown type */
		if (ostype == IFM_INST_ANY)
			continue;

		/* only add each media type to the list once */
		if (ice_is_bit_set(already_added, ostype))
			continue;

		ifmedia_add(media, IFM_ETHER | ostype, 0, NULL);
		ice_set_bit(ostype, already_added);
	}

	phy_high = le64toh(pcaps.phy_type_high);
	for (bit = 0; bit < 64; bit++) {
		uint64_t type = BIT_ULL(bit);
		uint64_t ostype;

		if ((phy_high & type) == 0)
			continue;

		/* get the OS media type */
		ostype = ice_get_phy_type_high(type);

		/* don't bother adding the unknown type */
		if (ostype == IFM_INST_ANY)
			continue;

		/* only add each media type to the list once */
		if (ice_is_bit_set(already_added, ostype))
			continue;

		ifmedia_add(media, IFM_ETHER | ostype, 0, NULL);
		ice_set_bit(ostype, already_added);
	}

	/* Use autoselect media by default */
	ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL);
	ifmedia_set(media, IFM_ETHER | IFM_AUTO);

	return (ICE_SUCCESS);
}

/**
 * ice_is_fw_health_report_supported
 * @hw: pointer to the hardware structure
 *
 * Return true if firmware supports health status reports,
 * false otherwise
 */
bool
ice_is_fw_health_report_supported(struct ice_hw *hw)
{
	if (hw->api_maj_ver > ICE_FW_API_HEALTH_REPORT_MAJ)
		return true;

	if (hw->api_maj_ver == ICE_FW_API_HEALTH_REPORT_MAJ) {
		if (hw->api_min_ver > ICE_FW_API_HEALTH_REPORT_MIN)
			return true;
		if (hw->api_min_ver == ICE_FW_API_HEALTH_REPORT_MIN &&
		    hw->api_patch >= ICE_FW_API_HEALTH_REPORT_PATCH)
			return true;
	}

	return false;
}

/**
 * ice_init_device_features - Init device driver features
 * @sc: driver softc structure
 *
 * @pre assumes that the function capabilities bits have been set up by
 * ice_init_hw().
 */
void
ice_init_device_features(struct ice_softc *sc)
{
	struct ice_hw *hw = &sc->hw;

	/* Set capabilities that all devices support */
	ice_set_bit(ICE_FEATURE_SRIOV, sc->feat_cap);
	ice_set_bit(ICE_FEATURE_RSS, sc->feat_cap);
	ice_set_bit(ICE_FEATURE_RDMA, sc->feat_cap);
	ice_set_bit(ICE_FEATURE_LENIENT_LINK_MODE, sc->feat_cap);
	ice_set_bit(ICE_FEATURE_LINK_MGMT_VER_1, sc->feat_cap);
	ice_set_bit(ICE_FEATURE_LINK_MGMT_VER_2, sc->feat_cap);
	ice_set_bit(ICE_FEATURE_HEALTH_STATUS, sc->feat_cap);
	ice_set_bit(ICE_FEATURE_FW_LOGGING, sc->feat_cap);
	ice_set_bit(ICE_FEATURE_HAS_PBA, sc->feat_cap);
	ice_set_bit(ICE_FEATURE_DCB, sc->feat_cap);
	ice_set_bit(ICE_FEATURE_TX_BALANCE, sc->feat_cap);

	/* Disable features due to hardware limitations... */
	if (!hw->func_caps.common_cap.rss_table_size)
		ice_clear_bit(ICE_FEATURE_RSS, sc->feat_cap);
	if (!hw->func_caps.common_cap.iwarp /* || !ice_enable_irdma */)
		ice_clear_bit(ICE_FEATURE_RDMA, sc->feat_cap);
	if (!hw->func_caps.common_cap.dcb)
		ice_clear_bit(ICE_FEATURE_DCB, sc->feat_cap);
	/* Disable features due to firmware limitations... */
	if (!ice_is_fw_health_report_supported(hw))
		ice_clear_bit(ICE_FEATURE_HEALTH_STATUS, sc->feat_cap);
	if (!ice_fwlog_supported(hw))
		ice_clear_bit(ICE_FEATURE_FW_LOGGING, sc->feat_cap);
	if (hw->fwlog_cfg.options & ICE_FWLOG_OPTION_IS_REGISTERED) {
		if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_FW_LOGGING))
			ice_set_bit(ICE_FEATURE_FW_LOGGING, sc->feat_en);
		else
			ice_fwlog_unregister(hw);
	}
#if 0
	/* Disable capabilities not supported by the OS */
	ice_disable_unsupported_features(sc->feat_cap);

	/* RSS is always enabled for iflib */
	if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_RSS))
		ice_set_bit(ICE_FEATURE_RSS, sc->feat_en);

	/* Disable features based on sysctl settings */
	if (!ice_tx_balance_en)
		ice_clear_bit(ICE_FEATURE_TX_BALANCE, sc->feat_cap);
#endif

	if (hw->dev_caps.supported_sensors & ICE_SENSOR_SUPPORT_E810_INT_TEMP) {
		ice_set_bit(ICE_FEATURE_TEMP_SENSOR, sc->feat_cap);
		ice_set_bit(ICE_FEATURE_TEMP_SENSOR, sc->feat_en);
	}
}

/**
 * ice_aq_send_driver_ver
 * @hw: pointer to the HW struct
 * @dv: driver's major, minor version
 * @cd: pointer to command details structure or NULL
 *
 * Send the driver version (0x0002) to the firmware
 */
enum ice_status
ice_aq_send_driver_ver(struct ice_hw *hw, struct ice_driver_ver *dv,
		       struct ice_sq_cd *cd)
{
	struct ice_aqc_driver_ver *cmd;
	struct ice_aq_desc desc;
	uint16_t len;

	cmd = &desc.params.driver_ver;

	if (!dv)
		return ICE_ERR_PARAM;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_driver_ver);

	desc.flags |= htole16(ICE_AQ_FLAG_RD);
	cmd->major_ver = dv->major_ver;
	cmd->minor_ver = dv->minor_ver;
	cmd->build_ver = dv->build_ver;
	cmd->subbuild_ver = dv->subbuild_ver;

	len = strlen(dv->driver_string);

	return ice_aq_send_cmd(hw, &desc, dv->driver_string, len, cd);
}

/**
 * ice_send_version - Send driver version to firmware
 * @sc: the device private softc
 *
 * Send the driver version to the firmware. This must be called as early as
 * possible after ice_init_hw().
 */
int
ice_send_version(struct ice_softc *sc)
{
	struct ice_driver_ver driver_version = {0};
	struct ice_hw *hw = &sc->hw;
	enum ice_status status;

	driver_version.major_ver = ice_major_version;
	driver_version.minor_ver = ice_minor_version;
	driver_version.build_ver = ice_patch_version;
	driver_version.subbuild_ver = ice_rc_version;

	strlcpy((char *)driver_version.driver_string, ice_driver_version,
		sizeof(driver_version.driver_string));

	status = ice_aq_send_driver_ver(hw, &driver_version, NULL);
	if (status) {
		printf("%s: Unable to send driver version to firmware, "
		    "err %s aq_err %s\n", sc->sc_dev.dv_xname,
		    ice_status_str(status),
		    ice_aq_str(hw->adminq.sq_last_status));
		return (EIO);
	}

	return (0);
}

void
ice_reinit_hw(struct ice_softc *sc)
{
	struct ice_hw *hw = &sc->hw;

	ice_deinit_hw(hw);

	printf("%s: not implemented\n", __func__);
}

/**
 * ice_cfg_tx_topo - Initialize new tx topology if available
 * @hw: pointer to the HW struct
 * @buf: pointer to Tx topology buffer
 * @len: buffer size
 *
 * The function will apply the new Tx topology from the package buffer
 * if available.
 */
enum ice_status
ice_cfg_tx_topo(struct ice_hw *hw, uint8_t *buf, uint32_t len)
{
#if 0
	u8 *current_topo, *new_topo = NULL;
	struct ice_run_time_cfg_seg *seg;
	struct ice_buf_hdr *section;
	struct ice_pkg_hdr *pkg_hdr;
	enum ice_ddp_state state;
	u16 i, size = 0, offset;
	enum ice_status status;
	u32 reg = 0;
	u8 flags;

	if (!buf || !len)
		return ICE_ERR_PARAM;

	/* Does FW support new Tx topology mode ? */
	if (!hw->func_caps.common_cap.tx_sched_topo_comp_mode_en) {
		ice_debug(hw, ICE_DBG_INIT, "FW doesn't support compatibility mode\n");
		return ICE_ERR_NOT_SUPPORTED;
	}

	current_topo = (u8 *)ice_malloc(hw, ICE_AQ_MAX_BUF_LEN);
	if (!current_topo)
		return ICE_ERR_NO_MEMORY;

	/* get the current Tx topology */
	status = ice_get_set_tx_topo(hw, current_topo, ICE_AQ_MAX_BUF_LEN, NULL,
				     &flags, false);
	ice_free(hw, current_topo);

	if (status) {
		ice_debug(hw, ICE_DBG_INIT, "Get current topology is failed\n");
		return status;
	}

	/* Is default topology already applied ? */
	if (!(flags & ICE_AQC_TX_TOPO_FLAGS_LOAD_NEW) &&
	    hw->num_tx_sched_layers == 9) {
		ice_debug(hw, ICE_DBG_INIT, "Loaded default topology\n");
		/* Already default topology is loaded */
		return ICE_ERR_ALREADY_EXISTS;
	}

	/* Is new topology already applied ? */
	if ((flags & ICE_AQC_TX_TOPO_FLAGS_LOAD_NEW) &&
	    hw->num_tx_sched_layers == 5) {
		ice_debug(hw, ICE_DBG_INIT, "Loaded new topology\n");
		/* Already new topology is loaded */
		return ICE_ERR_ALREADY_EXISTS;
	}

	/* Is set topology issued already ? */
	if (flags & ICE_AQC_TX_TOPO_FLAGS_ISSUED) {
		ice_debug(hw, ICE_DBG_INIT, "Update tx topology was done by another PF\n");
		/* add a small delay before exiting */
		for (i = 0; i < 20; i++)
			ice_msec_delay(100, true);
		return ICE_ERR_ALREADY_EXISTS;
	}

	/* Change the topology from new to default (5 to 9) */
	if (!(flags & ICE_AQC_TX_TOPO_FLAGS_LOAD_NEW) &&
	    hw->num_tx_sched_layers == 5) {
		ice_debug(hw, ICE_DBG_INIT, "Change topology from 5 to 9 layers\n");
		goto update_topo;
	}

	pkg_hdr = (struct ice_pkg_hdr *)buf;
	state = ice_verify_pkg(pkg_hdr, len);
	if (state) {
		ice_debug(hw, ICE_DBG_INIT, "failed to verify pkg (err: %d)\n",
			  state);
		return ICE_ERR_CFG;
	}

	/* find run time configuration segment */
	seg = (struct ice_run_time_cfg_seg *)
		ice_find_seg_in_pkg(hw, SEGMENT_TYPE_ICE_RUN_TIME_CFG, pkg_hdr);
	if (!seg) {
		ice_debug(hw, ICE_DBG_INIT, "5 layer topology segment is missing\n");
		return ICE_ERR_CFG;
	}

	if (LE32_TO_CPU(seg->buf_table.buf_count) < ICE_MIN_S_COUNT) {
		ice_debug(hw, ICE_DBG_INIT, "5 layer topology segment count(%d) is wrong\n",
			  seg->buf_table.buf_count);
		return ICE_ERR_CFG;
	}

	section = ice_pkg_val_buf(seg->buf_table.buf_array);

	if (!section || LE32_TO_CPU(section->section_entry[0].type) !=
		ICE_SID_TX_5_LAYER_TOPO) {
		ice_debug(hw, ICE_DBG_INIT, "5 layer topology section type is wrong\n");
		return ICE_ERR_CFG;
	}

	size = LE16_TO_CPU(section->section_entry[0].size);
	offset = LE16_TO_CPU(section->section_entry[0].offset);
	if (size < ICE_MIN_S_SZ || size > ICE_MAX_S_SZ) {
		ice_debug(hw, ICE_DBG_INIT, "5 layer topology section size is wrong\n");
		return ICE_ERR_CFG;
	}

	/* make sure the section fits in the buffer */
	if (offset + size > ICE_PKG_BUF_SIZE) {
		ice_debug(hw, ICE_DBG_INIT, "5 layer topology buffer > 4K\n");
		return ICE_ERR_CFG;
	}

	/* Get the new topology buffer */
	new_topo = ((u8 *)section) + offset;

update_topo:
	/* acquire global lock to make sure that set topology issued
	 * by one PF
	 */
	status = ice_acquire_res(hw, ICE_GLOBAL_CFG_LOCK_RES_ID, ICE_RES_WRITE,
				 ICE_GLOBAL_CFG_LOCK_TIMEOUT);
	if (status) {
		ice_debug(hw, ICE_DBG_INIT, "Failed to acquire global lock\n");
		return status;
	}

	/* check reset was triggered already or not */
	reg = rd32(hw, GLGEN_RSTAT);
	if (reg & GLGEN_RSTAT_DEVSTATE_M) {
		/* Reset is in progress, re-init the hw again */
		ice_debug(hw, ICE_DBG_INIT, "Reset is in progress. layer topology might be applied already\n");
		ice_check_reset(hw);
		return ICE_SUCCESS;
	}

	/* set new topology */
	status = ice_get_set_tx_topo(hw, new_topo, size, NULL, NULL, true);
	if (status) {
		ice_debug(hw, ICE_DBG_INIT, "Set tx topology is failed\n");
		return status;
	}

	/* new topology is updated, delay 1 second before issuing the CORRER */
	for (i = 0; i < 10; i++)
		ice_msec_delay(100, true);
	ice_reset(hw, ICE_RESET_CORER);
	/* CORER will clear the global lock, so no explicit call
	 * required for release
	 */
	return ICE_SUCCESS;
#else
	return ICE_ERR_NOT_IMPL;
#endif
}

/**
 * pkg_ver_empty - Check if a package version is empty
 * @pkg_ver: the package version to check
 * @pkg_name: the package name to check
 *
 * Checks if the package version structure is empty. We consider a package
 * version as empty if none of the versions are non-zero and the name string
 * is null as well.
 *
 * This is used to check if the package version was initialized by the driver,
 * as we do not expect an actual DDP package file to have a zero'd version and
 * name.
 *
 * @returns true if the package version is valid, or false otherwise.
 */
bool
pkg_ver_empty(struct ice_pkg_ver *pkg_ver, uint8_t *pkg_name)
{
	return (pkg_name[0] == '\0' &&
		pkg_ver->major == 0 &&
		pkg_ver->minor == 0 &&
		pkg_ver->update == 0 &&
		pkg_ver->draft == 0);
}

/**
 * ice_active_pkg_version_str - Format active package version info into a buffer
 * @hw: device hw structure
 * @buf: string buffer to store name/version string
 *
 * Formats the name and version of the active DDP package info into a string
 * buffer for use.
 */
void
ice_active_pkg_version_str(struct ice_hw *hw, char *buf, size_t bufsize)
{
	char name_buf[ICE_PKG_NAME_SIZE];

	/* If the active DDP package info is empty, use "None" */
	if (pkg_ver_empty(&hw->active_pkg_ver, hw->active_pkg_name)) {
		snprintf(buf, bufsize, "None");
		return;
	}

	/*
	 * This should already be null-terminated, but since this is a raw
	 * value from an external source, strlcpy() into a new buffer to
	 * make sure.
	 */
	strlcpy(name_buf, (char *)hw->active_pkg_name, bufsize);

	snprintf(buf, bufsize, "%s version %u.%u.%u.%u, track id 0x%08x",
	    name_buf,
	    hw->active_pkg_ver.major,
	    hw->active_pkg_ver.minor,
	    hw->active_pkg_ver.update,
	    hw->active_pkg_ver.draft,
	    hw->active_track_id);
}

/**
 * ice_os_pkg_version_str - Format OS package version info into a buffer
 * @hw: device hw structure
 * @buf: string buffer to store name/version string
 *
 * Formats the name and version of the OS DDP package as found in the ice_ddp
 * module into a string.
 *
 * @remark This will almost always be the same as the active package, but
 * could be different in some cases. Use ice_active_pkg_version_str to get the
 * version of the active DDP package.
 */
void
ice_os_pkg_version_str(struct ice_hw *hw, char *buf, size_t bufsize)
{
	char name_buf[ICE_PKG_NAME_SIZE];

	/* If the OS DDP package info is empty, use "None" */
	if (pkg_ver_empty(&hw->pkg_ver, hw->pkg_name)) {
		snprintf(buf, bufsize, "None");
		return;
	}

	/*
	 * This should already be null-terminated, but since this is a raw
	 * value from an external source, strlcpy() into a new buffer to
	 * make sure.
	 */
	strlcpy(name_buf, (char *)hw->pkg_name, bufsize);

	snprintf(buf, bufsize, "%s version %u.%u.%u.%u",
	    name_buf,
	    hw->pkg_ver.major,
	    hw->pkg_ver.minor,
	    hw->pkg_ver.update,
	    hw->pkg_ver.draft);
}

/**
 * ice_is_init_pkg_successful - check if DDP init was successful
 * @state: state of the DDP pkg after download
 */
bool
ice_is_init_pkg_successful(enum ice_ddp_state state)
{
	switch (state) {
	case ICE_DDP_PKG_SUCCESS:
	case ICE_DDP_PKG_SAME_VERSION_ALREADY_LOADED:
	case ICE_DDP_PKG_COMPATIBLE_ALREADY_LOADED:
		return true;
	default:
		return false;
	}
}

/**
 * ice_pkg_ver_compatible - Check if the package version is compatible
 * @pkg_ver: the package version to check
 *
 * Compares the package version number to the driver's expected major/minor
 * version. Returns an integer indicating whether the version is older, newer,
 * or compatible with the driver.
 *
 * @returns 0 if the package version is compatible, -1 if the package version
 * is older, and 1 if the package version is newer than the driver version.
 */
int
ice_pkg_ver_compatible(struct ice_pkg_ver *pkg_ver)
{
	if (pkg_ver->major > ICE_PKG_SUPP_VER_MAJ)
		return (1); /* newer */
	else if ((pkg_ver->major == ICE_PKG_SUPP_VER_MAJ) &&
		 (pkg_ver->minor > ICE_PKG_SUPP_VER_MNR))
		return (1); /* newer */
	else if ((pkg_ver->major == ICE_PKG_SUPP_VER_MAJ) &&
		 (pkg_ver->minor == ICE_PKG_SUPP_VER_MNR))
		return (0); /* compatible */
	else
		return (-1); /* older */
}

/**
 * ice_log_pkg_init - Log a message about status of DDP initialization
 * @sc: the device softc pointer
 * @pkg_status: the status result of ice_copy_and_init_pkg
 *
 * Called by ice_load_pkg after an attempt to download the DDP package
 * contents to the device to log an appropriate message for the system
 * administrator about download status.
 *
 * @post ice_is_init_pkg_successful function is used to determine
 * whether the download was successful and DDP package is compatible
 * with this driver. Otherwise driver will transition to Safe Mode.
 */
void
ice_log_pkg_init(struct ice_softc *sc, enum ice_ddp_state pkg_status)
{
	struct ice_hw *hw = &sc->hw;
	char active_pkg[ICE_PKG_NAME_SIZE];
	char os_pkg[ICE_PKG_NAME_SIZE];

	ice_active_pkg_version_str(hw, active_pkg, sizeof(active_pkg));
	ice_os_pkg_version_str(hw, os_pkg, sizeof(os_pkg));

	switch (pkg_status) {
	case ICE_DDP_PKG_SUCCESS:
		DPRINTF("%s: The DDP package was successfully loaded: %s.\n",
		    __func__, active_pkg);
		break;
	case ICE_DDP_PKG_SAME_VERSION_ALREADY_LOADED:
	case ICE_DDP_PKG_ALREADY_LOADED:
		DPRINTF("%s: DDP package already present on device: %s.\n",
		    __func__, active_pkg);
		break;
	case ICE_DDP_PKG_COMPATIBLE_ALREADY_LOADED:
		DPRINTF("%s: The driver could not load the DDP package file "
		    "because a compatible DDP package is already present on "
		    "the device.  The device has package %s.  The ice-ddp "
		    "file has package: %s.\n", __func__, active_pkg, os_pkg);
		break;
	case ICE_DDP_PKG_FILE_VERSION_TOO_HIGH:
		printf("%s: The device has a DDP package that is higher than "
		    "the driver supports.  The device has package %s. The "
		    "driver requires version %d.%d.x.x.  Entering Safe Mode.\n",
		    sc->sc_dev.dv_xname, active_pkg, ICE_PKG_SUPP_VER_MAJ,
		    ICE_PKG_SUPP_VER_MNR);
		break;
	case ICE_DDP_PKG_FILE_VERSION_TOO_LOW:
		printf("%s: The device has a DDP package that is lower than "
		    "the driver supports.  The device has package %s.  The "
		    "driver requires version %d.%d.x.x.  Entering Safe Mode.\n",
		    sc->sc_dev.dv_xname, active_pkg, ICE_PKG_SUPP_VER_MAJ,
		    ICE_PKG_SUPP_VER_MNR);
		break;
	case ICE_DDP_PKG_ALREADY_LOADED_NOT_SUPPORTED:
		/*
		 * This assumes that the active_pkg_ver will not be
		 * initialized if the ice_ddp package version is not
		 * supported.
		 */
		if (pkg_ver_empty(&hw->active_pkg_ver, hw->active_pkg_name)) {
			/* The ice_ddp version is not supported */
			if (ice_pkg_ver_compatible(&hw->pkg_ver) > 0) {
				DPRINTF("%s: The DDP package in the ice-ddp file "
				    "is higher than the driver supports.  The "
				    "ice-ddp file has package %s. The driver "
				    "requires version %d.%d.x.x.  Please use "
				    "an updated driver.  Entering Safe Mode.\n",
				    __func__, os_pkg, ICE_PKG_SUPP_VER_MAJ,
				    ICE_PKG_SUPP_VER_MNR);
			} else if (ice_pkg_ver_compatible(&hw->pkg_ver) < 0) {
				DPRINTF("%s: The DDP package in the "
				    "ice-ddp file is lower than the driver "
				    "supports. The ice_ddp module has package "
				    "%s.  The driver requires version "
				    "%d.%d.x.x. Please use an updated "
				    "ice-ddp file. Entering Safe Mode.\n",
				    __func__, os_pkg, ICE_PKG_SUPP_VER_MAJ,
				    ICE_PKG_SUPP_VER_MNR);
			} else {
				printf("%s: An unknown error occurred when "
				    "loading the DDP package.  The ice-ddp "
				    "file has package %s. The device has "
				    "package %s.  The driver requires version "
				    "%d.%d.x.x.  Entering Safe Mode.\n",
				    sc->sc_dev.dv_xname, os_pkg, active_pkg,
				    ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR);
			}
		} else {
			if (ice_pkg_ver_compatible(&hw->active_pkg_ver) > 0) {
				DPRINTF("%s: The device has a DDP package "
				    "that is higher than the driver supports. " 
				    "The device has package %s.  The driver "
				    "requires version %d.%d.x.x.  Entering "
				    "Safe Mode.\n", __func__, active_pkg,
				    ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR);
			} else if (ice_pkg_ver_compatible(&hw->active_pkg_ver)
			    < 0) {
				DPRINTF("%s: The device has a DDP package that "
				    "is lower than the driver supports.  The "
				    "device has package %s. The driver "
				    "requires version %d.%d.x.x.  "
				    "Entering Safe Mode.\n", __func__,
				    active_pkg, ICE_PKG_SUPP_VER_MAJ,
				    ICE_PKG_SUPP_VER_MNR);
			} else {
				printf("%s: An unknown error occurred when "
				    "loading the DDP package.  The ice-ddp "
				    "file has package %s.  The device has "
				    "package %s.  The driver requires "
				    "version %d.%d.x.x.  Entering Safe Mode.\n",
				    sc->sc_dev.dv_xname, os_pkg, active_pkg,
				    ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR);
			}
		}
		break;
	case ICE_DDP_PKG_INVALID_FILE:
		printf("%s: The DDP package in the ice-ddp file is invalid. "
		    "Entering Safe Mode\n", sc->sc_dev.dv_xname);
		break;
	case ICE_DDP_PKG_FW_MISMATCH:
		printf("%s: The firmware loaded on the device is not "
		    "compatible with the DDP package. "
		    "Please update the device's NVM.  Entering safe mode.\n",
		    sc->sc_dev.dv_xname);
		break;
	case ICE_DDP_PKG_NO_SEC_MANIFEST:
	case ICE_DDP_PKG_FILE_SIGNATURE_INVALID:
		printf("%s: The DDP package in the ice-ddp file cannot be "
		    "loaded because its signature is not valid.  Please "
		    "use a valid ice-ddp file.  Entering Safe Mode.\n",
		    sc->sc_dev.dv_xname);
		break;
	case ICE_DDP_PKG_SECURE_VERSION_NBR_TOO_LOW:
		printf("%s: The DDP package in the ice-ddp file could not "
		    "be loaded because its security revision is too low. "
		    "Please use an updated ice-ddp file.  "
		    "Entering Safe Mode.\n", sc->sc_dev.dv_xname);
		break;
	case ICE_DDP_PKG_MANIFEST_INVALID:
	case ICE_DDP_PKG_BUFFER_INVALID:
		printf("%s: An error occurred on the device while loading "
		    "the DDP package.  Entering Safe Mode.\n",
		    sc->sc_dev.dv_xname);
		break;
	default:
		printf("%s: An unknown error occurred when loading the "
		    "DDP package.  Entering Safe Mode.\n",
		    sc->sc_dev.dv_xname);
		break;
	}
}

/**
 * ice_copy_and_init_pkg - initialize/download a copy of the package
 * @hw: pointer to the hardware structure
 * @buf: pointer to the package buffer
 * @len: size of the package buffer
 *
 * This function copies the package buffer, and then calls ice_init_pkg() to
 * initialize the copied package contents.
 *
 * The copying is necessary if the package buffer supplied is constant, or if
 * the memory may disappear shortly after calling this function.
 *
 * If the package buffer resides in the data segment and can be modified, the
 * caller is free to use ice_init_pkg() instead of ice_copy_and_init_pkg().
 *
 * However, if the package buffer needs to be copied first, such as when being
 * read from a file, the caller should use ice_copy_and_init_pkg().
 *
 * This function will first copy the package buffer, before calling
 * ice_init_pkg(). The caller is free to immediately destroy the original
 * package buffer, as the new copy will be managed by this function and
 * related routines.
 */
enum ice_ddp_state
ice_copy_and_init_pkg(struct ice_hw *hw, const uint8_t *buf, uint32_t len)
{
#if 0
	enum ice_ddp_state state;
	u8 *buf_copy;

	if (!buf || !len)
		return ICE_DDP_PKG_ERR;

	buf_copy = (u8 *)ice_memdup(hw, buf, len, ICE_NONDMA_TO_NONDMA);

	state = ice_init_pkg(hw, buf_copy, len);
	if (!ice_is_init_pkg_successful(state)) {
		/* Free the copy, since we failed to initialize the package */
		ice_free(hw, buf_copy);
	} else {
		/* Track the copied pkg so we can free it later */
		hw->pkg_copy = buf_copy;
		hw->pkg_size = len;
	}
	return state;
#else
	return ICE_DDP_PKG_ERR;
#endif
}

/**
 * ice_load_pkg_file - Load the DDP package file using firmware_get
 * @sc: device private softc
 *
 * Use firmware_get to load the DDP package memory and then request that
 * firmware download the package contents and program the relevant hardware
 * bits.
 *
 * This function makes a copy of the DDP package memory which is tracked in
 * the ice_hw structure. The copy will be managed and released by
 * ice_deinit_hw(). This allows the firmware reference to be immediately
 * released using firmware_put.
 */
enum ice_status
ice_load_pkg_file(struct ice_softc *sc)
{
	struct ice_hw *hw = &sc->hw;
	enum ice_ddp_state state;
	uint8_t *pkg;
	size_t pkg_size;
	enum ice_status status = ICE_SUCCESS;
	uint8_t cached_layer_count;
	const char *fwname = "ice-ddp";
	int err;

	err = loadfirmware(fwname, &pkg, &pkg_size);
	if (err) {
		printf("%s: could not read firmware %s (error %d); "
		    "entering safe mode\n", sc->sc_dev.dv_xname, fwname, err);
		status = ICE_ERR_CFG;
		goto err_load_pkg;
	}

	/* Check for topology change */
	if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_TX_BALANCE)) {
		cached_layer_count = hw->num_tx_sched_layers;
		status = ice_cfg_tx_topo(&sc->hw, pkg, pkg_size);
		/* Success indicates a change was made */
		if (status == ICE_SUCCESS) {
			/* 9 -> 5 */
			if (cached_layer_count == 9)
				DPRINTF("%s: Transmit balancing feature "
				    "enabled\n", __func__);
			else
				DPRINTF("%s: Transmit balancing feature "
				    "disabled\n", __func__);
			ice_set_bit(ICE_FEATURE_TX_BALANCE, sc->feat_en);
			free(pkg, M_DEVBUF, pkg_size);
			return (status);
		} else if (status == ICE_ERR_CFG) {
			/* DDP does not support transmit balancing */
			DPRINTF("%s: DDP package does not support transmit balancing "
			    "feature - please update to the latest DDP package and "
			    "try again\n", __func__);
		}
	}

	/* Copy and download the pkg contents */
	state = ice_copy_and_init_pkg(hw, (const uint8_t *)pkg, pkg_size);

	/* Release the firmware reference */
	free(pkg, M_DEVBUF, pkg_size);

	/* Check the active DDP package version and log a message */
	ice_log_pkg_init(sc, state);

	/* Place the driver into safe mode */
	if (ice_is_init_pkg_successful(state))
		return (ICE_ERR_ALREADY_EXISTS);

err_load_pkg:
	ice_zero_bitmap(sc->feat_cap, ICE_FEATURE_COUNT);
	ice_zero_bitmap(sc->feat_en, ICE_FEATURE_COUNT);
	ice_set_bit(ICE_FEATURE_SAFE_MODE, sc->feat_cap);
	ice_set_bit(ICE_FEATURE_SAFE_MODE, sc->feat_en);

	return (status);
}

/**
 * ice_aq_set_event_mask
 * @hw: pointer to the HW struct
 * @port_num: port number of the physical function
 * @mask: event mask to be set
 * @cd: pointer to command details structure or NULL
 *
 * Set event mask (0x0613)
 */
enum ice_status
ice_aq_set_event_mask(struct ice_hw *hw, uint8_t port_num, uint16_t mask,
		      struct ice_sq_cd *cd)
{
	struct ice_aqc_set_event_mask *cmd;
	struct ice_aq_desc desc;

	cmd = &desc.params.set_event_mask;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_set_event_mask);

	cmd->lport_num = port_num;

	cmd->event_mask = htole16(mask);
	return ice_aq_send_cmd(hw, &desc, NULL, 0, cd);
}

/**
 * ice_init_link_events - Initialize Link Status Events mask
 * @sc: the device softc
 *
 * Initialize the Link Status Events mask to disable notification of link
 * events we don't care about in software. Also request that link status
 * events be enabled.
 */
int
ice_init_link_events(struct ice_softc *sc)
{
	struct ice_hw *hw = &sc->hw;
	enum ice_status status;
	uint16_t wanted_events;

	/* Set the bits for the events that we want to be notified by */
	wanted_events = (ICE_AQ_LINK_EVENT_UPDOWN |
			 ICE_AQ_LINK_EVENT_MEDIA_NA |
			 ICE_AQ_LINK_EVENT_MODULE_QUAL_FAIL);

	/* request that every event except the wanted events be masked */
	status = ice_aq_set_event_mask(hw, hw->port_info->lport, ~wanted_events, NULL);
	if (status) {
		printf("%s: Failed to set link status event mask, "
		    "err %s aq_err %s\n", sc->sc_dev.dv_xname,
		    ice_status_str(status),
		    ice_aq_str(hw->adminq.sq_last_status));
		return (EIO);
	}

	/* Request link info with the LSE bit set to enable link status events */
	status = ice_aq_get_link_info(hw->port_info, true, NULL, NULL);
	if (status) {
		printf("%s: Failed to enable link status events, "
		    "err %s aq_err %s\n", sc->sc_dev.dv_xname,
		    ice_status_str(status),
		    ice_aq_str(hw->adminq.sq_last_status));
		return (EIO);
	}

	return (0);
}

/**
 * ice_pkg_get_supported_vlan_mode - chk if DDP supports Double VLAN mode (DVM)
 * @hw: pointer to the HW struct
 * @dvm: output variable to determine if DDP supports DVM(true) or SVM(false)
 */
enum ice_status
ice_pkg_get_supported_vlan_mode(struct ice_hw *hw, bool *dvm)
{
#if 0
	u16 meta_init_size = sizeof(struct ice_meta_init_section);
	struct ice_meta_init_section *sect;
	struct ice_buf_build *bld;
	enum ice_status status;

	/* if anything fails, we assume there is no DVM support */
	*dvm = false;

	bld = ice_pkg_buf_alloc_single_section(hw,
					       ICE_SID_RXPARSER_METADATA_INIT,
					       meta_init_size, (void **)&sect);
	if (!bld)
		return ICE_ERR_NO_MEMORY;

	/* only need to read a single section */
	sect->count = CPU_TO_LE16(1);
	sect->offset = CPU_TO_LE16(ICE_META_VLAN_MODE_ENTRY);

	status = ice_aq_upload_section(hw,
				       (struct ice_buf_hdr *)ice_pkg_buf(bld),
				       ICE_PKG_BUF_SIZE, NULL);
	if (!status) {
		ice_declare_bitmap(entry, ICE_META_INIT_BITS);
		u32 arr[ICE_META_INIT_DW_CNT];
		u16 i;

		/* convert to host bitmap format */
		for (i = 0; i < ICE_META_INIT_DW_CNT; i++)
			arr[i] = LE32_TO_CPU(sect->entry[0].bm[i]);

		ice_bitmap_from_array32(entry, arr, (u16)ICE_META_INIT_BITS);

		/* check if DVM is supported */
		*dvm = ice_is_bit_set(entry, ICE_META_VLAN_MODE_BIT);
	}

	ice_pkg_buf_free(hw, bld);

	return status;
#else
	return ICE_ERR_NOT_IMPL;
#endif
}

/**
 * ice_pkg_supports_dvm - find out if DDP supports DVM
 * @hw: pointer to the HW structure
 */
bool
ice_pkg_supports_dvm(struct ice_hw *hw)
{
	enum ice_status status;
	bool pkg_supports_dvm;

	status = ice_pkg_get_supported_vlan_mode(hw, &pkg_supports_dvm);
	if (status) {
		DNPRINTF(ICE_DBG_PKG,
		     "Failed to get supported VLAN mode, status %d\n", status);
		return false;
	}

	return pkg_supports_dvm;
}

/**
 * ice_aq_get_vlan_mode - get the VLAN mode of the device
 * @hw: pointer to the HW structure
 * @get_params: structure FW fills in based on the current VLAN mode config
 *
 * Get VLAN Mode Parameters (0x020D)
 */
enum ice_status
ice_aq_get_vlan_mode(struct ice_hw *hw,
		     struct ice_aqc_get_vlan_mode *get_params)
{
	struct ice_aq_desc desc;

	if (!get_params)
		return ICE_ERR_PARAM;

	ice_fill_dflt_direct_cmd_desc(&desc,
				      ice_aqc_opc_get_vlan_mode_parameters);

	return ice_aq_send_cmd(hw, &desc, get_params, sizeof(*get_params),
			       NULL);
}

/**
 * ice_fw_supports_dvm - find out if FW supports DVM
 * @hw: pointer to the HW structure
 */
bool
ice_fw_supports_dvm(struct ice_hw *hw)
{
	struct ice_aqc_get_vlan_mode get_vlan_mode = { 0 };
	enum ice_status status;

	/* If firmware returns success, then it supports DVM, else it only
	 * supports SVM
	 */
	status = ice_aq_get_vlan_mode(hw, &get_vlan_mode);
	if (status) {
		DNPRINTF(ICE_DBG_NVM,
		    "%s: Failed to get VLAN mode, status %d\n",
		    __func__, status);
		return false;
	}

	return true;
}

/**
 * ice_is_dvm_supported - check if Double VLAN Mode is supported
 * @hw: pointer to the hardware structure
 *
 * Returns true if Double VLAN Mode (DVM) is supported and false if only Single
 * VLAN Mode (SVM) is supported. In order for DVM to be supported the DDP and
 * firmware must support it, otherwise only SVM is supported. This function
 * should only be called while the global config lock is held and after the
 * package has been successfully downloaded.
 */
bool
ice_is_dvm_supported(struct ice_hw *hw)
{
	if (!ice_pkg_supports_dvm(hw)) {
		DNPRINTF(ICE_DBG_PKG, "DDP doesn't support DVM\n");
		return false;
	}

	if (!ice_fw_supports_dvm(hw)) {
		DNPRINTF(ICE_DBG_PKG, "FW doesn't support DVM\n");
		return false;
	}

	return true;
}

/**
 * ice_aq_set_port_params - set physical port parameters
 * @pi: pointer to the port info struct
 * @bad_frame_vsi: defines the VSI to which bad frames are forwarded
 * @save_bad_pac: if set packets with errors are forwarded to the bad frames VSI
 * @pad_short_pac: if set transmit packets smaller than 60 bytes are padded
 * @double_vlan: if set double VLAN is enabled
 * @cd: pointer to command details structure or NULL
 *
 * Set Physical port parameters (0x0203)
 */
enum ice_status
ice_aq_set_port_params(struct ice_port_info *pi, uint16_t bad_frame_vsi,
		       bool save_bad_pac, bool pad_short_pac, bool double_vlan,
		       struct ice_sq_cd *cd)
{
	struct ice_aqc_set_port_params *cmd;
	struct ice_hw *hw = pi->hw;
	struct ice_aq_desc desc;
	uint16_t cmd_flags = 0;

	cmd = &desc.params.set_port_params;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_set_port_params);
	cmd->bad_frame_vsi = htole16(bad_frame_vsi);
	if (save_bad_pac)
		cmd_flags |= ICE_AQC_SET_P_PARAMS_SAVE_BAD_PACKETS;
	if (pad_short_pac)
		cmd_flags |= ICE_AQC_SET_P_PARAMS_PAD_SHORT_PACKETS;
	if (double_vlan)
		cmd_flags |= ICE_AQC_SET_P_PARAMS_DOUBLE_VLAN_ENA;
	cmd->cmd_flags = htole16(cmd_flags);

	return ice_aq_send_cmd(hw, &desc, NULL, 0, cd);
}

/**
 * ice_aq_set_vlan_mode - set the VLAN mode of the device
 * @hw: pointer to the HW structure
 * @set_params: requested VLAN mode configuration
 *
 * Set VLAN Mode Parameters (0x020C)
 */
enum ice_status
ice_aq_set_vlan_mode(struct ice_hw *hw,
		     struct ice_aqc_set_vlan_mode *set_params)
{
	uint8_t rdma_packet, mng_vlan_prot_id;
	struct ice_aq_desc desc;

	if (!set_params)
		return ICE_ERR_PARAM;

	if (set_params->l2tag_prio_tagging > ICE_AQ_VLAN_PRIO_TAG_MAX)
		return ICE_ERR_PARAM;

	rdma_packet = set_params->rdma_packet;
	if (rdma_packet != ICE_AQ_SVM_VLAN_RDMA_PKT_FLAG_SETTING &&
	    rdma_packet != ICE_AQ_DVM_VLAN_RDMA_PKT_FLAG_SETTING)
		return ICE_ERR_PARAM;

	mng_vlan_prot_id = set_params->mng_vlan_prot_id;
	if (mng_vlan_prot_id != ICE_AQ_VLAN_MNG_PROTOCOL_ID_OUTER &&
	    mng_vlan_prot_id != ICE_AQ_VLAN_MNG_PROTOCOL_ID_INNER)
		return ICE_ERR_PARAM;

	ice_fill_dflt_direct_cmd_desc(&desc,
				      ice_aqc_opc_set_vlan_mode_parameters);
	desc.flags |= htole16(ICE_AQ_FLAG_RD);

	return ice_aq_send_cmd(hw, &desc, set_params, sizeof(*set_params),
			       NULL);
}

/**
 * ice_set_svm - set single VLAN mode
 * @hw: pointer to the HW structure
 */
enum ice_status
ice_set_svm(struct ice_hw *hw)
{
	struct ice_aqc_set_vlan_mode *set_params;
	enum ice_status status;

	status = ice_aq_set_port_params(hw->port_info, 0,
	    false, false, false, NULL);
	if (status) {
		DNPRINTF(ICE_DBG_INIT,
		    "Failed to set port parameters for single VLAN mode\n");
		return status;
	}

	set_params = (struct ice_aqc_set_vlan_mode *)
		ice_malloc(hw, sizeof(*set_params));
	if (!set_params)
		return ICE_ERR_NO_MEMORY;

	/* default configuration for SVM configurations */
	set_params->l2tag_prio_tagging = ICE_AQ_VLAN_PRIO_TAG_INNER_CTAG;
	set_params->rdma_packet = ICE_AQ_SVM_VLAN_RDMA_PKT_FLAG_SETTING;
	set_params->mng_vlan_prot_id = ICE_AQ_VLAN_MNG_PROTOCOL_ID_INNER;

	status = ice_aq_set_vlan_mode(hw, set_params);
	if (status)
		DNPRINTF(ICE_DBG_INIT,
		    "Failed to configure port in single VLAN mode\n");

	ice_free(hw, set_params);
	return status;
}

/**
 * ice_set_vlan_mode
 * @hw: pointer to the HW structure
 */
enum ice_status
ice_set_vlan_mode(struct ice_hw *hw)
{
	if (!ice_is_dvm_supported(hw))
		return ICE_SUCCESS;

	return ice_set_svm(hw);
}

/**
 * ice_nvm_version_str - Format the NVM version information into a sbuf
 * @hw: device hw structure
 * @buf: string buffer to store version string
 *
 * Formats the NVM information including firmware version, API version, NVM
 * version, the EETRACK id, and OEM specific version information into a string
 * buffer.
 */
const char *
ice_nvm_version_str(struct ice_hw *hw, char *buf, size_t bufsize)
{
	struct ice_nvm_info *nvm = &hw->flash.nvm;
	struct ice_orom_info *orom = &hw->flash.orom;
	struct ice_netlist_info *netlist = &hw->flash.netlist;

	/* Note that the netlist versions are stored in packed Binary Coded
	 * Decimal format. The use of '%x' will correctly display these as
	 * decimal numbers. This works because every 4 bits will be displayed
	 * as a hexadecimal digit, and the BCD format will only use the values
	 * 0-9.
	 */
	snprintf(buf, bufsize,
		    "fw %u.%u.%u api %u.%u nvm %x.%02x etid %08x netlist %x.%x.%x-%x.%x.%x.%04x oem %u.%u.%u",
		    hw->fw_maj_ver, hw->fw_min_ver, hw->fw_patch,
		    hw->api_maj_ver, hw->api_min_ver,
		    nvm->major, nvm->minor, nvm->eetrack,
		    netlist->major, netlist->minor,
		    netlist->type >> 16, netlist->type & 0xFFFF,
		    netlist->rev, netlist->cust_ver, netlist->hash,
		    orom->major, orom->build, orom->patch);

	return buf;
}

/**
 * ice_print_nvm_version - Print the NVM info to the kernel message log
 * @sc: the device softc structure
 *
 * Format and print an NVM version string using ice_nvm_version_str().
 */
void
ice_print_nvm_version(struct ice_softc *sc)
{
	struct ice_hw *hw = &sc->hw;
	static char buf[512];

	printf("%s: %s, address %s\n", sc->sc_dev.dv_xname,
	    ice_nvm_version_str(hw, buf, sizeof(buf)),
	    ether_sprintf(hw->port_info->mac.perm_addr));
}

/**
 * ice_setup_scctx - Setup the softc context structure; in the FreeBSD
 * driver this function sets up a context used by iflib. Instead, the
 * OpenBSD driver uses it to initialize softc fields which depend on
 * driver mode and hw capabilities.
 */
void
ice_setup_scctx(struct ice_softc *sc)
{
	struct ice_hw *hw = &sc->hw;
	bool safe_mode, recovery_mode;
	int i;

	safe_mode = ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE);
	recovery_mode = ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE);

	/*
	 * If the driver loads in Safe mode or Recovery mode, limit iflib to
	 * a single queue pair.
	 */
	if (safe_mode || recovery_mode) {
		sc->isc_ntxqsets = sc->isc_nrxqsets = 1;
		sc->isc_ntxqsets_max = 1;
		sc->isc_nrxqsets_max = 1;
	} else {
		sc->isc_ntxqsets = hw->func_caps.common_cap.rss_table_size;
		sc->isc_nrxqsets = hw->func_caps.common_cap.rss_table_size;

		sc->isc_ntxqsets_max = hw->func_caps.common_cap.num_txq;
		sc->isc_nrxqsets_max = hw->func_caps.common_cap.num_rxq;
	}

	sc->isc_txqsizes[0] = roundup(sc->isc_ntxd[0]
	    * sizeof(struct ice_tx_desc), DBA_ALIGN);
	sc->isc_rxqsizes[0] = roundup(sc->isc_nrxd[0]
	    * sizeof(union ice_32b_rx_flex_desc), DBA_ALIGN);

	sc->isc_tx_nsegments = ICE_MAX_TX_SEGS;
	sc->isc_tx_tso_segments_max = ICE_MAX_TSO_SEGS;
	sc->isc_tx_tso_size_max = ICE_TSO_SIZE;
	sc->isc_tx_tso_segsize_max = ICE_MAX_DMA_SEG_SIZE;
#if 0
	sc->isc_msix_bar = pci_msix_table_bar(dev);
#endif
	sc->isc_rss_table_size = hw->func_caps.common_cap.rss_table_size;
#if 0
	/*
	 * If the driver loads in recovery mode, disable Tx/Rx functionality
	 */
	if (recovery_mode)
		scctx->isc_txrx = &ice_recovery_txrx;
	else
		scctx->isc_txrx = &ice_txrx;
#endif
	/*
	 * If the driver loads in Safe mode or Recovery mode, disable
	 * advanced features including hardware offloads.
	 */
	if (safe_mode || recovery_mode) {
		sc->isc_capenable = ICE_SAFE_CAPS;
		sc->isc_tx_csum_flags = 0;
	} else {
		sc->isc_capenable = ICE_FULL_CAPS;
#if 0
		sc->isc_tx_csum_flags = ICE_CSUM_OFFLOAD;
#endif
	}

	sc->isc_capabilities = sc->isc_capenable;

	for (i = 0; i < nitems(sc->isc_ntxd); i++)
		sc->isc_ntxd[i] = ICE_DEFAULT_DESC_COUNT;
	for (i = 0; i < nitems(sc->isc_nrxd); i++)
		sc->isc_nrxd[i] = ICE_DEFAULT_DESC_COUNT;


} /* ice_setup_scctx */

/**
 * ice_resmgr_init - Initialize a resource manager structure
 * @resmgr: structure to track the resource manager state
 * @num_res: the maximum number of resources it can assign
 *
 * Initialize the state of a resource manager structure, allocating space to
 * assign up to the requested number of resources. Uses bit strings to track
 * which resources have been assigned. This type of resmgr is intended to be
 * used for tracking LAN queue assignments between VSIs.
 */
int
ice_resmgr_init(struct ice_resmgr *resmgr, uint16_t num_res)
{
	resmgr->resources = ice_bit_alloc(num_res);
	if (resmgr->resources == NULL)
		return (ENOMEM);

	resmgr->num_res = num_res;
	resmgr->contig_only = false;
	return (0);
}

/**
 * ice_resmgr_init_contig_only - Initialize a resource manager structure
 * @resmgr: structure to track the resource manager state
 * @num_res: the maximum number of resources it can assign
 *
 * Functions similarly to ice_resmgr_init(), but the resulting resmgr structure
 * will only allow contiguous allocations. This type of resmgr is intended to
 * be used with tracking device MSI-X interrupt allocations.
 */
int
ice_resmgr_init_contig_only(struct ice_resmgr *resmgr, uint16_t num_res)
{
	int error;

	error = ice_resmgr_init(resmgr, num_res);
	if (error)
		return (error);

	resmgr->contig_only = true;
	return (0);
}

/**
 * ice_resmgr_destroy - Deallocate memory associated with a resource manager
 * @resmgr: resource manager structure
 *
 * De-allocates the bit string associated with this resource manager. It is
 * expected that this function will not be called until all of the assigned
 * resources have been released.
 */
void
ice_resmgr_destroy(struct ice_resmgr *resmgr)
{
	if (resmgr->resources != NULL) {
		int set;

		set = ice_bit_count(resmgr->resources, 0, resmgr->num_res);
		KASSERT(set == 0);

		free(resmgr->resources, M_DEVBUF,
		    ice_bitstr_size(resmgr->num_res));
		resmgr->resources = NULL;
	}
	resmgr->num_res = 0;
}

/**
 * ice_resmgr_assign_contiguous - Assign contiguous mapping of resources
 * @resmgr: resource manager structure
 * @idx: memory to store mapping, at least num_res wide
 * @num_res: the number of resources to assign
 *
 * Assign num_res number of contiguous resources into the idx mapping. On
 * success, idx will be updated to map each index to a PF resource.
 *
 * This function guarantees that the resource mapping will be contiguous, and
 * will fail if that is not possible.
 */
int
ice_resmgr_assign_contiguous(struct ice_resmgr *resmgr, uint16_t *idx,
    uint16_t num_res)
{
	int start, i;

	ice_bit_ffc_area(resmgr->resources, resmgr->num_res, num_res, &start);
	if (start < 0)
		return (ENOSPC);

	/* Set each bit and update the index array */
	for (i = 0; i < num_res; i++) {
		ice_bit_set(resmgr->resources, start + i);
		idx[i] = start + i;
	}

	return (0);
}

/**
 * ice_alloc_intr_tracking - Setup interrupt tracking structures
 * @sc: device softc structure
 *
 * Sets up the resource manager for keeping track of interrupt allocations,
 * and initializes the tracking maps for the PF's interrupt allocations.
 *
 * Unlike the scheme for queues, this is done in one step since both the
 * manager and the maps both have the same lifetime.
 *
 * @returns 0 on success, or an error code on failure.
 */
int
ice_alloc_intr_tracking(struct ice_softc *sc)
{
	struct ice_hw *hw = &sc->hw;
	size_t nvec;
	int err, i;

	nvec = hw->func_caps.common_cap.num_msix_vectors;

	/* Initialize the interrupt allocation manager */
	err = ice_resmgr_init_contig_only(&sc->dev_imgr, nvec);
	if (err) {
		printf("%s: Unable to initialize PF interrupt manager: "
		    "error %d\n", sc->sc_dev.dv_xname, err);
		return (err);
	}

	/* Allocate PF interrupt mapping storage */
	sc->pf_imap = (uint16_t *)mallocarray(nvec, sizeof(uint16_t),
	    M_DEVBUF, M_NOWAIT);
	if (sc->pf_imap == NULL) {
		printf("%s: Unable to allocate PF imap memory\n",
		    sc->sc_dev.dv_xname);
		err = ENOMEM;
		goto free_imgr;
	}
#if 0
	sc->rdma_imap = (uint16_t *)mallocarray(nvec, sizeof(uint16_t),
	    M_DEVBUF, M_NOWAIT);
	if (sc->sc_rdma_imap == NULL) {
		printf("%s: Unable to allocate RDMA imap memory\n",
		    sc->sc_dev.dv_xname);
		err = ENOMEM;
		goto free_pf_imap;
	}
#endif
	for (i = 0; i < nvec; i++) {
		sc->pf_imap[i] = ICE_INVALID_RES_IDX;
#if 0
		sc->rdma_imap[i] = ICE_INVALID_RES_IDX;
#endif
	}

	return (0);
#if 0
free_pf_imap:
	free(sc->pf_imap, M_DEVBUF, nvec * sizeof(uint16_t));
	sc->pf_imap = NULL;
#endif
free_imgr:
	ice_resmgr_destroy(&sc->dev_imgr);
	return (err);
}

/**
 * ice_resmgr_release_map - Release previously assigned resource mapping
 * @resmgr: the resource manager structure
 * @idx: previously assigned resource mapping
 * @num_res: number of resources in the mapping
 *
 * Clears the assignment of each resource in the provided resource index.
 * Updates the idx to indicate that each of the virtual indexes have
 * invalid resource mappings by assigning them to ICE_INVALID_RES_IDX.
 */
void
ice_resmgr_release_map(struct ice_resmgr *resmgr, uint16_t *idx,
    uint16_t num_res)
{
	int i;

	for (i = 0; i < num_res; i++) {
		if (idx[i] < resmgr->num_res)
			ice_bit_clear(resmgr->resources, idx[i]);
		idx[i] = ICE_INVALID_RES_IDX;
	}
}

/**
 * ice_free_intr_tracking - Free PF interrupt tracking structures
 * @sc: device softc structure
 *
 * Frees the interrupt resource allocation manager and the PF's owned maps.
 *
 * VF maps are released when the owning VF's are destroyed, which should always
 * happen before this function is called.
 */
void
ice_free_intr_tracking(struct ice_softc *sc)
{
	if (sc->pf_imap) {
		ice_resmgr_release_map(&sc->dev_imgr, sc->pf_imap,
		    sc->lan_vectors);
		free(sc->pf_imap, M_DEVBUF,
		    sizeof(uint16_t) * sc->lan_vectors);
		sc->pf_imap = NULL;
	}
#if 0
	if (sc->rdma_imap) {
		ice_resmgr_release_map(&sc->dev_imgr, sc->rdma_imap,
		    sc->lan_vectors);
		free(sc->rdma_imap, M_DEVBUF,
		    sizeof(uint16_t) * sc->lan_vectors);
		sc->rdma_imap = NULL;
	}
#endif
	ice_resmgr_destroy(&sc->dev_imgr);

	ice_resmgr_destroy(&sc->os_imgr);
}

/**
 * ice_setup_vsi_common - Common VSI setup for both dynamic and static VSIs
 * @sc: the device private softc structure
 * @vsi: the VSI to setup
 * @type: the VSI type of the new VSI
 * @idx: the index in the all_vsi array to use
 * @dynamic: whether this VSI memory was dynamically allocated
 *
 * Perform setup for a VSI that is common to both dynamically allocated VSIs
 * and the static PF VSI which is embedded in the softc structure.
 */
void
ice_setup_vsi_common(struct ice_softc *sc, struct ice_vsi *vsi,
		     enum ice_vsi_type type, int idx, bool dynamic)
{
	struct ice_hw *hw = &sc->hw;

	/* Store important values in VSI struct */
	vsi->type = type;
	vsi->sc = sc;
	vsi->idx = idx;
	sc->all_vsi[idx] = vsi;
	vsi->dynamic = dynamic;

	/* Set default mirroring rule information */
	vsi->rule_mir_ingress = ICE_INVAL_MIRROR_RULE_ID;
	vsi->rule_mir_egress = ICE_INVAL_MIRROR_RULE_ID;
#if 0
	/* Setup the VSI tunables now */
	ice_add_vsi_tunables(vsi, sc->vsi_sysctls);
#endif
	vsi->mbuf_sz = MCLBYTES + ETHER_ALIGN;
	vsi->max_frame_size = hw->port_info->phy.link_info.max_frame_size;
}

/**
 * ice_setup_pf_vsi - Setup the PF VSI
 * @sc: the device private softc
 *
 * Setup the PF VSI structure which is embedded as sc->pf_vsi in the device
 * private softc. Unlike other VSIs, the PF VSI memory is allocated as part of
 * the softc memory, instead of being dynamically allocated at creation.
 */
void
ice_setup_pf_vsi(struct ice_softc *sc)
{
	ice_setup_vsi_common(sc, &sc->pf_vsi, ICE_VSI_PF, 0, false);
}

/**
 * ice_alloc_vsi_qmap
 * @vsi: VSI structure
 * @max_tx_queues: Number of transmit queues to identify
 * @max_rx_queues: Number of receive queues to identify
 *
 * Allocates a max_[t|r]x_queues array of words for the VSI where each
 * word contains the index of the queue it represents.  In here, all
 * words are initialized to an index of ICE_INVALID_RES_IDX, indicating
 * all queues for this VSI are not yet assigned an index and thus,
 * not ready for use.
 *
 * Returns an error code on failure.
 */
int
ice_alloc_vsi_qmap(struct ice_vsi *vsi, const int max_tx_queues,
		   const int max_rx_queues)
{
	struct ice_softc *sc = vsi->sc;
	int i;

	KASSERT(max_tx_queues > 0);
	KASSERT(max_rx_queues > 0);

	/* Allocate Tx queue mapping memory */
	vsi->tx_qmap = (uint16_t *)mallocarray(max_tx_queues, sizeof(uint16_t),
	    M_DEVBUF, M_WAITOK);
	if (!vsi->tx_qmap) {
		printf("%s: Unable to allocate Tx qmap memory\n",
		    sc->sc_dev.dv_xname);
		return (ENOMEM);
	}

	/* Allocate Rx queue mapping memory */
	vsi->rx_qmap = (uint16_t *) mallocarray(max_rx_queues, sizeof(uint16_t),
	    M_DEVBUF, M_WAITOK);
	if (!vsi->rx_qmap) {
		printf("%s: Unable to allocate Rx qmap memory\n",
		    sc->sc_dev.dv_xname);
		goto free_tx_qmap;
	}

	/* Mark every queue map as invalid to start with */
	for (i = 0; i < max_tx_queues; i++)
		vsi->tx_qmap[i] = ICE_INVALID_RES_IDX;
	for (i = 0; i < max_rx_queues; i++)
		vsi->rx_qmap[i] = ICE_INVALID_RES_IDX;

	return 0;

free_tx_qmap:
	free(vsi->tx_qmap, M_DEVBUF, max_tx_queues * sizeof(uint16_t));
	vsi->tx_qmap = NULL;

	return (ENOMEM);
}

/**
 * ice_vsig_find_vsi - find a VSIG that contains a specified VSI
 * @hw: pointer to the hardware structure
 * @blk: HW block
 * @vsi: VSI of interest
 * @vsig: pointer to receive the VSI group
 *
 * This function will lookup the VSI entry in the XLT2 list and return
 * the VSI group its associated with.
 */
enum ice_status
ice_vsig_find_vsi(struct ice_hw *hw, enum ice_block blk, uint16_t vsi,
    uint16_t *vsig)
{
	if (!vsig || vsi >= ICE_MAX_VSI)
		return ICE_ERR_PARAM;

	/* As long as there's a default or valid VSIG associated with the input
	 * VSI, the functions returns a success. Any handling of VSIG will be
	 * done by the following add, update or remove functions.
	 */
	*vsig = hw->blk[blk].xlt2.vsis[vsi].vsig;

	return ICE_SUCCESS;
}

/**
 * ice_vsig_prof_id_count - count profiles in a VSIG
 * @hw: pointer to the HW struct
 * @blk: hardware block
 * @vsig: VSIG to remove the profile from
 */
uint16_t
ice_vsig_prof_id_count(struct ice_hw *hw, enum ice_block blk,
    uint16_t vsig)
{
	uint16_t idx = vsig & ICE_VSIG_IDX_M, count = 0;
	struct ice_vsig_prof *p;

	TAILQ_FOREACH(p, &hw->blk[blk].xlt2.vsig_tbl[idx].prop_lst, list)
		count++;

	return count;
}

/**
 * ice_vsig_get_ref - returns number of VSIs belong to a VSIG
 * @hw: pointer to the hardware structure
 * @blk: HW block
 * @vsig: VSIG to query
 * @refs: pointer to variable to receive the reference count
 */
enum ice_status
ice_vsig_get_ref(struct ice_hw *hw, enum ice_block blk, uint16_t vsig,
    uint16_t *refs)
{
	uint16_t idx = vsig & ICE_VSIG_IDX_M;
	struct ice_vsig_vsi *ptr;

	*refs = 0;

	if (!hw->blk[blk].xlt2.vsig_tbl[idx].in_use)
		return ICE_ERR_DOES_NOT_EXIST;

	ptr = hw->blk[blk].xlt2.vsig_tbl[idx].first_vsi;
	while (ptr) {
		(*refs)++;
		ptr = ptr->next_vsi;
	}

	return ICE_SUCCESS;
}

/* Key creation */

#define ICE_DC_KEY	0x1	/* don't care */
#define ICE_DC_KEYINV	0x1
#define ICE_NM_KEY	0x0	/* never match */
#define ICE_NM_KEYINV	0x0
#define ICE_0_KEY	0x1	/* match 0 */
#define ICE_0_KEYINV	0x0
#define ICE_1_KEY	0x0	/* match 1 */
#define ICE_1_KEYINV	0x1

/**
 * ice_gen_key_word - generate 16-bits of a key/mask word
 * @val: the value
 * @valid: valid bits mask (change only the valid bits)
 * @dont_care: don't care mask
 * @nvr_mtch: never match mask
 * @key: pointer to an array of where the resulting key portion
 * @key_inv: pointer to an array of where the resulting key invert portion
 *
 * This function generates 16-bits from a 8-bit value, an 8-bit don't care mask
 * and an 8-bit never match mask. The 16-bits of output are divided into 8 bits
 * of key and 8 bits of key invert.
 *
 *     '0' =    b01, always match a 0 bit
 *     '1' =    b10, always match a 1 bit
 *     '?' =    b11, don't care bit (always matches)
 *     '~' =    b00, never match bit
 *
 * Input:
 *          val:         b0  1  0  1  0  1
 *          dont_care:   b0  0  1  1  0  0
 *          never_mtch:  b0  0  0  0  1  1
 *          ------------------------------
 * Result:  key:        b01 10 11 11 00 00
 */
enum ice_status
ice_gen_key_word(uint8_t val, uint8_t valid, uint8_t dont_care,
    uint8_t nvr_mtch, uint8_t *key, uint8_t *key_inv)
{
	uint8_t in_key = *key, in_key_inv = *key_inv;
	uint8_t i;

	/* 'dont_care' and 'nvr_mtch' masks cannot overlap */
	if ((dont_care ^ nvr_mtch) != (dont_care | nvr_mtch))
		return ICE_ERR_CFG;

	*key = 0;
	*key_inv = 0;

	/* encode the 8 bits into 8-bit key and 8-bit key invert */
	for (i = 0; i < 8; i++) {
		*key >>= 1;
		*key_inv >>= 1;

		if (!(valid & 0x1)) { /* change only valid bits */
			*key |= (in_key & 0x1) << 7;
			*key_inv |= (in_key_inv & 0x1) << 7;
		} else if (dont_care & 0x1) { /* don't care bit */
			*key |= ICE_DC_KEY << 7;
			*key_inv |= ICE_DC_KEYINV << 7;
		} else if (nvr_mtch & 0x1) { /* never match bit */
			*key |= ICE_NM_KEY << 7;
			*key_inv |= ICE_NM_KEYINV << 7;
		} else if (val & 0x01) { /* exact 1 match */
			*key |= ICE_1_KEY << 7;
			*key_inv |= ICE_1_KEYINV << 7;
		} else { /* exact 0 match */
			*key |= ICE_0_KEY << 7;
			*key_inv |= ICE_0_KEYINV << 7;
		}

		dont_care >>= 1;
		nvr_mtch >>= 1;
		valid >>= 1;
		val >>= 1;
		in_key >>= 1;
		in_key_inv >>= 1;
	}

	return ICE_SUCCESS;
}

/**
 * ice_bits_max_set - determine if the number of bits set is within a maximum
 * @mask: pointer to the byte array which is the mask
 * @size: the number of bytes in the mask
 * @max: the max number of set bits
 *
 * This function determines if there are at most 'max' number of bits set in an
 * array. Returns true if the number for bits set is <= max or will return false
 * otherwise.
 */
bool
ice_bits_max_set(const uint8_t *mask, uint16_t size, uint16_t max)
{
	uint16_t count = 0;
	uint16_t i;

	/* check each byte */
	for (i = 0; i < size; i++) {
		/* if 0, go to next byte */
		if (!mask[i])
			continue;

		/* We know there is at least one set bit in this byte because of
		 * the above check; if we already have found 'max' number of
		 * bits set, then we can return failure now.
		 */
		if (count == max)
			return false;

		/* count the bits in this byte, checking threshold */
		count += ice_popcount16(mask[i]);
		if (count > max)
			return false;
	}

	return true;
}

/**
 * ice_set_key - generate a variable sized key with multiples of 16-bits
 * @key: pointer to where the key will be stored
 * @size: the size of the complete key in bytes (must be even)
 * @val: array of 8-bit values that makes up the value portion of the key
 * @upd: array of 8-bit masks that determine what key portion to update
 * @dc: array of 8-bit masks that make up the don't care mask
 * @nm: array of 8-bit masks that make up the never match mask
 * @off: the offset of the first byte in the key to update
 * @len: the number of bytes in the key update
 *
 * This function generates a key from a value, a don't care mask and a never
 * match mask.
 * upd, dc, and nm are optional parameters, and can be NULL:
 *	upd == NULL --> upd mask is all 1's (update all bits)
 *	dc == NULL --> dc mask is all 0's (no don't care bits)
 *	nm == NULL --> nm mask is all 0's (no never match bits)
 */
enum ice_status
ice_set_key(uint8_t *key, uint16_t size, uint8_t *val, uint8_t *upd,
    uint8_t *dc, uint8_t *nm, uint16_t off, uint16_t len)
{
	uint16_t half_size;
	uint16_t i;

	/* size must be a multiple of 2 bytes. */
	if (size % 2)
		return ICE_ERR_CFG;
	half_size = size / 2;

	if (off + len > half_size)
		return ICE_ERR_CFG;

	/* Make sure at most one bit is set in the never match mask. Having more
	 * than one never match mask bit set will cause HW to consume excessive
	 * power otherwise; this is a power management efficiency check.
	 */
#define ICE_NVR_MTCH_BITS_MAX	1
	if (nm && !ice_bits_max_set(nm, len, ICE_NVR_MTCH_BITS_MAX))
		return ICE_ERR_CFG;

	for (i = 0; i < len; i++)
		if (ice_gen_key_word(val[i], upd ? upd[i] : 0xff,
				     dc ? dc[i] : 0, nm ? nm[i] : 0,
				     key + off + i, key + half_size + off + i))
			return ICE_ERR_CFG;

	return ICE_SUCCESS;
}

/**
 * ice_prof_gen_key - generate profile ID key
 * @hw: pointer to the HW struct
 * @blk: the block in which to write profile ID to
 * @ptg: packet type group (PTG) portion of key
 * @vsig: VSIG portion of key
 * @cdid: CDID portion of key
 * @flags: flag portion of key
 * @vl_msk: valid mask
 * @dc_msk: don't care mask
 * @nm_msk: never match mask
 * @key: output of profile ID key
 */
enum ice_status
ice_prof_gen_key(struct ice_hw *hw, enum ice_block blk, uint8_t ptg,
    uint16_t vsig, uint8_t cdid, uint16_t flags,
    uint8_t vl_msk[ICE_TCAM_KEY_VAL_SZ],
    uint8_t dc_msk[ICE_TCAM_KEY_VAL_SZ],
    uint8_t nm_msk[ICE_TCAM_KEY_VAL_SZ],
    uint8_t key[ICE_TCAM_KEY_SZ])
{
	struct ice_prof_id_key inkey;

	inkey.xlt1 = ptg;
	inkey.xlt2_cdid = htole16(vsig);
	inkey.flags = htole16(flags);

	switch (hw->blk[blk].prof.cdid_bits) {
	case 0:
		break;
	case 2:
#define ICE_CD_2_M 0xC000U
#define ICE_CD_2_S 14
		inkey.xlt2_cdid &= ~htole16(ICE_CD_2_M);
		inkey.xlt2_cdid |= htole16(BIT(cdid) << ICE_CD_2_S);
		break;
	case 4:
#define ICE_CD_4_M 0xF000U
#define ICE_CD_4_S 12
		inkey.xlt2_cdid &= ~htole16(ICE_CD_4_M);
		inkey.xlt2_cdid |= htole16(BIT(cdid) << ICE_CD_4_S);
		break;
	case 8:
#define ICE_CD_8_M 0xFF00U
#define ICE_CD_8_S 16
		inkey.xlt2_cdid &= ~htole16(ICE_CD_8_M);
		inkey.xlt2_cdid |= htole16(BIT(cdid) << ICE_CD_8_S);
		break;
	default:
		DNPRINTF(ICE_DBG_PKG, "%s: Error in profile config\n",
		    __func__);
		break;
	}

	return ice_set_key(key, ICE_TCAM_KEY_SZ, (uint8_t *)&inkey,
	    vl_msk, dc_msk, nm_msk, 0, ICE_TCAM_KEY_SZ / 2);
}

/**
 * ice_tcam_write_entry - write TCAM entry
 * @hw: pointer to the HW struct
 * @blk: the block in which to write profile ID to
 * @idx: the entry index to write to
 * @prof_id: profile ID
 * @ptg: packet type group (PTG) portion of key
 * @vsig: VSIG portion of key
 * @cdid: CDID portion of key
 * @flags: flag portion of key
 * @vl_msk: valid mask
 * @dc_msk: don't care mask
 * @nm_msk: never match mask
 */
enum ice_status
ice_tcam_write_entry(struct ice_hw *hw, enum ice_block blk, uint16_t idx,
		     uint8_t prof_id, uint8_t ptg, uint16_t vsig, uint8_t cdid, uint16_t flags,
		     uint8_t vl_msk[ICE_TCAM_KEY_VAL_SZ],
		     uint8_t dc_msk[ICE_TCAM_KEY_VAL_SZ],
		     uint8_t nm_msk[ICE_TCAM_KEY_VAL_SZ])
{
	struct ice_prof_tcam_entry;
	enum ice_status status;

	status = ice_prof_gen_key(hw, blk, ptg, vsig, cdid, flags, vl_msk,
				  dc_msk, nm_msk, hw->blk[blk].prof.t[idx].key);
	if (!status) {
		hw->blk[blk].prof.t[idx].addr = htole16(idx);
		hw->blk[blk].prof.t[idx].prof_id = prof_id;
	}

	return status;
}

/**
 * ice_tcam_ent_rsrc_type - get TCAM entry resource type for a block type
 * @blk: the block type
 * @rsrc_type: pointer to variable to receive the resource type
 */
bool
ice_tcam_ent_rsrc_type(enum ice_block blk, uint16_t *rsrc_type)
{
	switch (blk) {
	case ICE_BLK_RSS:
		*rsrc_type = ICE_AQC_RES_TYPE_HASH_PROF_BLDR_TCAM;
		break;
	case ICE_BLK_PE:
		*rsrc_type = ICE_AQC_RES_TYPE_QHASH_PROF_BLDR_TCAM;
		break;
	default:
		return false;
	}
	return true;
}

/**
 * ice_alloc_hw_res - allocate resource
 * @hw: pointer to the HW struct
 * @type: type of resource
 * @num: number of resources to allocate
 * @btm: allocate from bottom
 * @res: pointer to array that will receive the resources
 */
enum ice_status
ice_alloc_hw_res(struct ice_hw *hw, uint16_t type, uint16_t num, bool btm,
    uint16_t *res)
{
	struct ice_aqc_alloc_free_res_elem *buf;
	enum ice_status status;
	uint16_t buf_len;

	buf_len = ice_struct_size(buf, elem, num);
	buf = (struct ice_aqc_alloc_free_res_elem *)ice_malloc(hw, buf_len);
	if (!buf)
		return ICE_ERR_NO_MEMORY;

	/* Prepare buffer to allocate resource. */
	buf->num_elems = htole16(num);
	buf->res_type = htole16(type | ICE_AQC_RES_TYPE_FLAG_DEDICATED |
				    ICE_AQC_RES_TYPE_FLAG_IGNORE_INDEX);
	if (btm)
		buf->res_type |= htole16(ICE_AQC_RES_TYPE_FLAG_SCAN_BOTTOM);

	status = ice_aq_alloc_free_res(hw, 1, buf, buf_len,
				       ice_aqc_opc_alloc_res, NULL);
	if (status)
		goto ice_alloc_res_exit;

	memcpy(res, buf->elem, sizeof(*buf->elem) * num);

ice_alloc_res_exit:
	ice_free(hw, buf);
	return status;
}

/**
 * ice_free_hw_res - free allocated HW resource
 * @hw: pointer to the HW struct
 * @type: type of resource to free
 * @num: number of resources
 * @res: pointer to array that contains the resources to free
 */
enum ice_status
ice_free_hw_res(struct ice_hw *hw, uint16_t type, uint16_t num, uint16_t *res)
{
	struct ice_aqc_alloc_free_res_elem *buf;
	enum ice_status status;
	uint16_t buf_len;

	/* prevent overflow; all callers currently hard-code num as 1 */
	KASSERT(num == 1);

	buf_len = ice_struct_size(buf, elem, num);
	buf = (struct ice_aqc_alloc_free_res_elem *)ice_malloc(hw, buf_len);
	if (!buf)
		return ICE_ERR_NO_MEMORY;

	/* Prepare buffer to free resource. */
	buf->num_elems = htole16(num);
	buf->res_type = htole16(type);
	memcpy(buf->elem, res, sizeof(*buf->elem) * num);

	status = ice_aq_alloc_free_res(hw, num, buf, buf_len,
				       ice_aqc_opc_free_res, NULL);

	ice_free(hw, buf);
	return status;
}

/**
 * ice_free_tcam_ent - free hardware TCAM entry
 * @hw: pointer to the HW struct
 * @blk: the block from which to free the TCAM entry
 * @tcam_idx: the TCAM entry to free
 *
 * This function frees an entry in a Profile ID TCAM for a specific block.
 */
enum ice_status
ice_free_tcam_ent(struct ice_hw *hw, enum ice_block blk, uint16_t tcam_idx)
{
	uint16_t res_type;

	if (!ice_tcam_ent_rsrc_type(blk, &res_type))
		return ICE_ERR_PARAM;

	return ice_free_hw_res(hw, res_type, 1, &tcam_idx);
}

/**
 * ice_rel_tcam_idx - release a TCAM index
 * @hw: pointer to the HW struct
 * @blk: hardware block
 * @idx: the index to release
 */
enum ice_status
ice_rel_tcam_idx(struct ice_hw *hw, enum ice_block blk, uint16_t idx)
{
	/* Masks to invoke a never match entry */
	uint8_t vl_msk[ICE_TCAM_KEY_VAL_SZ] = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
	uint8_t dc_msk[ICE_TCAM_KEY_VAL_SZ] = { 0xFE, 0xFF, 0xFF, 0xFF, 0xFF };
	uint8_t nm_msk[ICE_TCAM_KEY_VAL_SZ] = { 0x01, 0x00, 0x00, 0x00, 0x00 };
	enum ice_status status;

	/* write the TCAM entry */
	status = ice_tcam_write_entry(hw, blk, idx, 0, 0, 0, 0, 0, vl_msk,
				      dc_msk, nm_msk);
	if (status)
		return status;

	/* release the TCAM entry */
	status = ice_free_tcam_ent(hw, blk, idx);

	return status;
}

/**
 * ice_rem_prof_id - remove one profile from a VSIG
 * @hw: pointer to the HW struct
 * @blk: hardware block
 * @prof: pointer to profile structure to remove
 */
enum ice_status
ice_rem_prof_id(struct ice_hw *hw, enum ice_block blk,
    struct ice_vsig_prof *prof)
{
	enum ice_status status;
	uint16_t i;

	for (i = 0; i < prof->tcam_count; i++)
		if (prof->tcam[i].in_use) {
			prof->tcam[i].in_use = false;
			status = ice_rel_tcam_idx(hw, blk,
						  prof->tcam[i].tcam_idx);
			if (status)
				return ICE_ERR_HW_TABLE;
		}

	return ICE_SUCCESS;
}

/**
 * ice_rem_vsig - remove VSIG
 * @hw: pointer to the HW struct
 * @blk: hardware block
 * @vsig: the VSIG to remove
 * @chg: the change list
 */
enum ice_status
ice_rem_vsig(struct ice_hw *hw, enum ice_block blk, uint16_t vsig,
    struct ice_chs_chg_head *chg)
{
	uint16_t idx = vsig & ICE_VSIG_IDX_M;
	struct ice_vsig_vsi *vsi_cur;
	struct ice_vsig_prof *d, *t;

	/* remove TCAM entries */
	TAILQ_FOREACH_SAFE(d, &hw->blk[blk].xlt2.vsig_tbl[idx].prop_lst,
	    list, t) {
		enum ice_status status;

		status = ice_rem_prof_id(hw, blk, d);
		if (status)
			return status;

		TAILQ_REMOVE(&hw->blk[blk].xlt2.vsig_tbl[idx].prop_lst,
		    d, list);
		ice_free(hw, d);
	}

	/* Move all VSIS associated with this VSIG to the default VSIG */
	vsi_cur = hw->blk[blk].xlt2.vsig_tbl[idx].first_vsi;
	/* If the VSIG has at least 1 VSI then iterate through the list
	 * and remove the VSIs before deleting the group.
	 */
	if (vsi_cur) {
		do {
			struct ice_vsig_vsi *tmp = vsi_cur->next_vsi;
			struct ice_chs_chg *p;

			p = (struct ice_chs_chg *)ice_malloc(hw, sizeof(*p));
			if (!p)
				return ICE_ERR_NO_MEMORY;

			p->type = ICE_VSIG_REM;
			p->orig_vsig = vsig;
			p->vsig = ICE_DEFAULT_VSIG;
			p->vsi = (uint16_t)(vsi_cur - hw->blk[blk].xlt2.vsis);

			TAILQ_INSERT_HEAD(chg, p, list_entry);

			vsi_cur = tmp;
		} while (vsi_cur);
	}

	return ice_vsig_free(hw, blk, vsig);
}

/**
 * ice_rem_prof_id_vsig - remove a specific profile from a VSIG
 * @hw: pointer to the HW struct
 * @blk: hardware block
 * @vsig: VSIG to remove the profile from
 * @hdl: profile handle indicating which profile to remove
 * @chg: list to receive a record of changes
 */
enum ice_status
ice_rem_prof_id_vsig(struct ice_hw *hw, enum ice_block blk, uint16_t vsig,
    uint64_t hdl, struct ice_chs_chg_head *chg)
{
	uint16_t idx = vsig & ICE_VSIG_IDX_M;
	struct ice_vsig_prof *p, *t;

	TAILQ_FOREACH_SAFE(p, &hw->blk[blk].xlt2.vsig_tbl[idx].prop_lst,
	    list, t) {
		if (p->profile_cookie == hdl) {
			enum ice_status status;

			if (ice_vsig_prof_id_count(hw, blk, vsig) == 1)
				/* this is the last profile, remove the VSIG */
				return ice_rem_vsig(hw, blk, vsig, chg);

			status = ice_rem_prof_id(hw, blk, p);
			if (!status) {
				TAILQ_REMOVE(
				    &hw->blk[blk].xlt2.vsig_tbl[idx].prop_lst,
				    p, list);
				ice_free(hw, p);
			}
			return status;
		}
	}

	return ICE_ERR_DOES_NOT_EXIST;
}

/**
 * ice_rem_chg_tcam_ent - remove a specific TCAM entry from change list
 * @hw: pointer to the HW struct
 * @idx: the index of the TCAM entry to remove
 * @chg: the list of change structures to search
 */
void
ice_rem_chg_tcam_ent(struct ice_hw *hw, uint16_t idx,
    struct ice_chs_chg_head *chg)
{
	struct ice_chs_chg *pos, *tmp;

	TAILQ_FOREACH_SAFE(pos, chg, list_entry, tmp) {
		if (pos->type == ICE_TCAM_ADD && pos->tcam_idx == idx) {
			TAILQ_REMOVE(chg, pos, list_entry);
			ice_free(hw, pos);
		}
	}
}

/**
 * ice_alloc_tcam_ent - allocate hardware TCAM entry
 * @hw: pointer to the HW struct
 * @blk: the block to allocate the TCAM for
 * @btm: true to allocate from bottom of table, false to allocate from top
 * @tcam_idx: pointer to variable to receive the TCAM entry
 *
 * This function allocates a new entry in a Profile ID TCAM for a specific
 * block.
 */
enum ice_status
ice_alloc_tcam_ent(struct ice_hw *hw, enum ice_block blk, bool btm,
		   uint16_t *tcam_idx)
{
	uint16_t res_type;

	if (!ice_tcam_ent_rsrc_type(blk, &res_type))
		return ICE_ERR_PARAM;

	return ice_alloc_hw_res(hw, res_type, 1, btm, tcam_idx);
}

/**
 * ice_prof_tcam_ena_dis - add enable or disable TCAM change
 * @hw: pointer to the HW struct
 * @blk: hardware block
 * @enable: true to enable, false to disable
 * @vsig: the VSIG of the TCAM entry
 * @tcam: pointer the TCAM info structure of the TCAM to disable
 * @chg: the change list
 *
 * This function appends an enable or disable TCAM entry in the change log
 */
enum ice_status
ice_prof_tcam_ena_dis(struct ice_hw *hw, enum ice_block blk, bool enable,
		      uint16_t vsig, struct ice_tcam_inf *tcam,
		      struct ice_chs_chg_head *chg)
{
	enum ice_status status;
	struct ice_chs_chg *p;
	uint8_t vl_msk[ICE_TCAM_KEY_VAL_SZ] = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
	uint8_t dc_msk[ICE_TCAM_KEY_VAL_SZ] = { 0xFF, 0xFF, 0x00, 0x00, 0x00 };
	uint8_t nm_msk[ICE_TCAM_KEY_VAL_SZ] = { 0x00, 0x00, 0x00, 0x00, 0x00 };

	/* if disabling, free the TCAM */
	if (!enable) {
		status = ice_rel_tcam_idx(hw, blk, tcam->tcam_idx);

		/* if we have already created a change for this TCAM entry, then
		 * we need to remove that entry, in order to prevent writing to
		 * a TCAM entry we no longer will have ownership of.
		 */
		ice_rem_chg_tcam_ent(hw, tcam->tcam_idx, chg);
		tcam->tcam_idx = 0;
		tcam->in_use = 0;
		return status;
	}

	/* for re-enabling, reallocate a TCAM */
	status = ice_alloc_tcam_ent(hw, blk, true, &tcam->tcam_idx);
	if (status)
		return status;

	/* add TCAM to change list */
	p = (struct ice_chs_chg *)ice_malloc(hw, sizeof(*p));
	if (!p)
		return ICE_ERR_NO_MEMORY;

	status = ice_tcam_write_entry(hw, blk, tcam->tcam_idx, tcam->prof_id,
				      tcam->ptg, vsig, 0, 0, vl_msk, dc_msk,
				      nm_msk);
	if (status)
		goto err_ice_prof_tcam_ena_dis;

	tcam->in_use = 1;

	p->type = ICE_TCAM_ADD;
	p->add_tcam_idx = true;
	p->prof_id = tcam->prof_id;
	p->ptg = tcam->ptg;
	p->vsig = 0;
	p->tcam_idx = tcam->tcam_idx;

	/* log change */
	TAILQ_INSERT_HEAD(chg, p, list_entry);

	return ICE_SUCCESS;

err_ice_prof_tcam_ena_dis:
	ice_free(hw, p);
	return status;
}

/**
 * ice_adj_prof_priorities - adjust profile based on priorities
 * @hw: pointer to the HW struct
 * @blk: hardware block
 * @vsig: the VSIG for which to adjust profile priorities
 * @chg: the change list
 */
enum ice_status
ice_adj_prof_priorities(struct ice_hw *hw, enum ice_block blk, uint16_t vsig,
    struct ice_chs_chg_head *chg)
{
	ice_declare_bitmap(ptgs_used, ICE_XLT1_CNT);
	enum ice_status status = ICE_SUCCESS;
	struct ice_vsig_prof *t;
	uint16_t idx;

	ice_zero_bitmap(ptgs_used, ICE_XLT1_CNT);
	idx = vsig & ICE_VSIG_IDX_M;

	/* Priority is based on the order in which the profiles are added. The
	 * newest added profile has highest priority and the oldest added
	 * profile has the lowest priority. Since the profile property list for
	 * a VSIG is sorted from newest to oldest, this code traverses the list
	 * in order and enables the first of each PTG that it finds (that is not
	 * already enabled); it also disables any duplicate PTGs that it finds
	 * in the older profiles (that are currently enabled).
	 */

	TAILQ_FOREACH(t, &hw->blk[blk].xlt2.vsig_tbl[idx].prop_lst, list) {
		uint16_t i;

		for (i = 0; i < t->tcam_count; i++) {
			bool used;

			/* Scan the priorities from newest to oldest.
			 * Make sure that the newest profiles take priority.
			 */
			used = ice_is_bit_set(ptgs_used, t->tcam[i].ptg);

			if (used && t->tcam[i].in_use) {
				/* need to mark this PTG as never match, as it
				 * was already in use and therefore duplicate
				 * (and lower priority)
				 */
				status = ice_prof_tcam_ena_dis(hw, blk, false,
							       vsig,
							       &t->tcam[i],
							       chg);
				if (status)
					return status;
			} else if (!used && !t->tcam[i].in_use) {
				/* need to enable this PTG, as it in not in use
				 * and not enabled (highest priority)
				 */
				status = ice_prof_tcam_ena_dis(hw, blk, true,
							       vsig,
							       &t->tcam[i],
							       chg);
				if (status)
					return status;
			}

			/* keep track of used ptgs */
			ice_set_bit(t->tcam[i].ptg, ptgs_used);
		}
	}

	return status;
}

/**
 * ice_get_profs_vsig - get a copy of the list of profiles from a VSIG
 * @hw: pointer to the HW struct
 * @blk: hardware block
 * @vsig: VSIG from which to copy the list
 * @lst: output list
 *
 * This routine makes a copy of the list of profiles in the specified VSIG.
 */
enum ice_status
ice_get_profs_vsig(struct ice_hw *hw, enum ice_block blk, uint16_t vsig,
		   struct ice_vsig_prof_head *lst)
{
	struct ice_vsig_prof *ent1, *ent2;
	uint16_t idx = vsig & ICE_VSIG_IDX_M;

	TAILQ_FOREACH(ent1, &hw->blk[blk].xlt2.vsig_tbl[idx].prop_lst, list) {
		struct ice_vsig_prof *p;

		/* copy to the input list */
		p = (struct ice_vsig_prof *)ice_memdup(hw, ent1, sizeof(*p));
		if (!p)
			goto err_ice_get_profs_vsig;

		TAILQ_INSERT_TAIL(lst, p, list);
	}

	return ICE_SUCCESS;

err_ice_get_profs_vsig:
	TAILQ_FOREACH_SAFE(ent1, lst, list, ent2) {
		TAILQ_REMOVE(lst, ent1, list);
		ice_free(hw, ent1);
	}

	return ICE_ERR_NO_MEMORY;
}

/**
 * ice_rem_prof_from_list - remove a profile from list
 * @hw: pointer to the HW struct
 * @lst: list to remove the profile from
 * @hdl: the profile handle indicating the profile to remove
 */
enum ice_status
ice_rem_prof_from_list(struct ice_hw *hw, struct ice_vsig_prof_head *lst,
    uint64_t hdl)
{
	struct ice_vsig_prof *ent, *tmp;

	TAILQ_FOREACH_SAFE(ent, lst, list, tmp) {
		if (ent->profile_cookie == hdl) {
			TAILQ_REMOVE(lst, ent, list);
			ice_free(hw, ent);
			return ICE_SUCCESS;
		}
	}	

	return ICE_ERR_DOES_NOT_EXIST;
}

/**
 * ice_vsig_remove_vsi - remove VSI from VSIG
 * @hw: pointer to the hardware structure
 * @blk: HW block
 * @vsi: VSI to remove
 * @vsig: VSI group to remove from
 *
 * The function will remove the input VSI from its VSI group and move it
 * to the DEFAULT_VSIG.
 */
enum ice_status
ice_vsig_remove_vsi(struct ice_hw *hw, enum ice_block blk, uint16_t vsi,
    uint16_t vsig)
{
	struct ice_vsig_vsi **vsi_head, *vsi_cur, *vsi_tgt;
	uint16_t idx;

	idx = vsig & ICE_VSIG_IDX_M;

	if (vsi >= ICE_MAX_VSI || idx >= ICE_MAX_VSIGS)
		return ICE_ERR_PARAM;

	if (!hw->blk[blk].xlt2.vsig_tbl[idx].in_use)
		return ICE_ERR_DOES_NOT_EXIST;

	/* entry already in default VSIG, don't have to remove */
	if (idx == ICE_DEFAULT_VSIG)
		return ICE_SUCCESS;

	vsi_head = &hw->blk[blk].xlt2.vsig_tbl[idx].first_vsi;
	if (!(*vsi_head))
		return ICE_ERR_CFG;

	vsi_tgt = &hw->blk[blk].xlt2.vsis[vsi];
	vsi_cur = (*vsi_head);

	/* iterate the VSI list, skip over the entry to be removed */
	while (vsi_cur) {
		if (vsi_tgt == vsi_cur) {
			(*vsi_head) = vsi_cur->next_vsi;
			break;
		}
		vsi_head = &vsi_cur->next_vsi;
		vsi_cur = vsi_cur->next_vsi;
	}

	/* verify if VSI was removed from group list */
	if (!vsi_cur)
		return ICE_ERR_DOES_NOT_EXIST;

	vsi_cur->vsig = ICE_DEFAULT_VSIG;
	vsi_cur->changed = 1;
	vsi_cur->next_vsi = NULL;

	return ICE_SUCCESS;
}

/**
 * ice_vsig_add_mv_vsi - add or move a VSI to a VSI group
 * @hw: pointer to the hardware structure
 * @blk: HW block
 * @vsi: VSI to move
 * @vsig: destination VSI group
 *
 * This function will move or add the input VSI to the target VSIG.
 * The function will find the original VSIG the VSI belongs to and
 * move the entry to the DEFAULT_VSIG, update the original VSIG and
 * then move entry to the new VSIG.
 */
enum ice_status
ice_vsig_add_mv_vsi(struct ice_hw *hw, enum ice_block blk, uint16_t vsi,
    uint16_t vsig)
{
	struct ice_vsig_vsi *tmp;
	enum ice_status status;
	uint16_t orig_vsig, idx;

	idx = vsig & ICE_VSIG_IDX_M;

	if (vsi >= ICE_MAX_VSI || idx >= ICE_MAX_VSIGS)
		return ICE_ERR_PARAM;

	/* if VSIG not in use and VSIG is not default type this VSIG
	 * doesn't exist.
	 */
	if (!hw->blk[blk].xlt2.vsig_tbl[idx].in_use &&
	    vsig != ICE_DEFAULT_VSIG)
		return ICE_ERR_DOES_NOT_EXIST;

	status = ice_vsig_find_vsi(hw, blk, vsi, &orig_vsig);
	if (status)
		return status;

	/* no update required if vsigs match */
	if (orig_vsig == vsig)
		return ICE_SUCCESS;

	if (orig_vsig != ICE_DEFAULT_VSIG) {
		/* remove entry from orig_vsig and add to default VSIG */
		status = ice_vsig_remove_vsi(hw, blk, vsi, orig_vsig);
		if (status)
			return status;
	}

	if (idx == ICE_DEFAULT_VSIG)
		return ICE_SUCCESS;

	/* Create VSI entry and add VSIG and prop_mask values */
	hw->blk[blk].xlt2.vsis[vsi].vsig = vsig;
	hw->blk[blk].xlt2.vsis[vsi].changed = 1;

	/* Add new entry to the head of the VSIG list */
	tmp = hw->blk[blk].xlt2.vsig_tbl[idx].first_vsi;
	hw->blk[blk].xlt2.vsig_tbl[idx].first_vsi =
		&hw->blk[blk].xlt2.vsis[vsi];
	hw->blk[blk].xlt2.vsis[vsi].next_vsi = tmp;
	hw->blk[blk].xlt2.t[vsi] = vsig;

	return ICE_SUCCESS;
}

/**
 * ice_move_vsi - move VSI to another VSIG
 * @hw: pointer to the HW struct
 * @blk: hardware block
 * @vsi: the VSI to move
 * @vsig: the VSIG to move the VSI to
 * @chg: the change list
 */
enum ice_status
ice_move_vsi(struct ice_hw *hw, enum ice_block blk, uint16_t vsi, uint16_t vsig,
	     struct ice_chs_chg_head *chg)
{
	enum ice_status status;
	struct ice_chs_chg *p;
	uint16_t orig_vsig;

	p = (struct ice_chs_chg *)ice_malloc(hw, sizeof(*p));
	if (!p)
		return ICE_ERR_NO_MEMORY;

	status = ice_vsig_find_vsi(hw, blk, vsi, &orig_vsig);
	if (!status)
		status = ice_vsig_add_mv_vsi(hw, blk, vsi, vsig);

	if (status) {
		ice_free(hw, p);
		return status;
	}

	p->type = ICE_VSI_MOVE;
	p->vsi = vsi;
	p->orig_vsig = orig_vsig;
	p->vsig = vsig;

	TAILQ_INSERT_HEAD(chg, p, list_entry);

	return ICE_SUCCESS;
}

/**
 * ice_vsig_alloc_val - allocate a new VSIG by value
 * @hw: pointer to the hardware structure
 * @blk: HW block
 * @vsig: the VSIG to allocate
 *
 * This function will allocate a given VSIG specified by the VSIG parameter.
 */
uint16_t
ice_vsig_alloc_val(struct ice_hw *hw, enum ice_block blk, uint16_t vsig)
{
	uint16_t idx = vsig & ICE_VSIG_IDX_M;

	if (!hw->blk[blk].xlt2.vsig_tbl[idx].in_use) {
		TAILQ_INIT(&hw->blk[blk].xlt2.vsig_tbl[idx].prop_lst);
		hw->blk[blk].xlt2.vsig_tbl[idx].in_use = true;
	}

	return ICE_VSIG_VALUE(idx, hw->pf_id);
}

/**
 * ice_vsig_alloc - Finds a free entry and allocates a new VSIG
 * @hw: pointer to the hardware structure
 * @blk: HW block
 *
 * This function will iterate through the VSIG list and mark the first
 * unused entry for the new VSIG entry as used and return that value.
 */
uint16_t
ice_vsig_alloc(struct ice_hw *hw, enum ice_block blk)
{
	uint16_t i;

	for (i = 1; i < ICE_MAX_VSIGS; i++)
		if (!hw->blk[blk].xlt2.vsig_tbl[i].in_use)
			return ice_vsig_alloc_val(hw, blk, i);

	return ICE_DEFAULT_VSIG;
}

/**
 * ice_has_prof_vsig - check to see if VSIG has a specific profile
 * @hw: pointer to the hardware structure
 * @blk: HW block
 * @vsig: VSIG to check against
 * @hdl: profile handle
 */
bool
ice_has_prof_vsig(struct ice_hw *hw, enum ice_block blk, uint16_t vsig,
    uint64_t hdl)
{
	uint16_t idx = vsig & ICE_VSIG_IDX_M;
	struct ice_vsig_prof *ent;

	TAILQ_FOREACH(ent, &hw->blk[blk].xlt2.vsig_tbl[idx].prop_lst, list) {
		if (ent->profile_cookie == hdl)
			return true;
	}

	DNPRINTF(ICE_DBG_INIT,
	    "%s: Characteristic list for VSI group %d not found\n",
	    __func__, vsig);

	return false;
}

/**
 * ice_search_prof_id - Search for a profile tracking ID
 * @hw: pointer to the HW struct
 * @blk: hardware block
 * @id: profile tracking ID
 *
 * This will search for a profile tracking ID which was previously added.
 * The profile map lock should be held before calling this function.
 */
struct ice_prof_map *
ice_search_prof_id(struct ice_hw *hw, enum ice_block blk, uint64_t id)
{
	struct ice_prof_map *entry = NULL;
	struct ice_prof_map *map;

	TAILQ_FOREACH(map, &hw->blk[blk].es.prof_map, list) {
		if (map->profile_cookie == id) {
			entry = map;
			break;
		}
	}

	return entry;
}

/**
 * ice_add_prof_id_vsig - add profile to VSIG
 * @hw: pointer to the HW struct
 * @blk: hardware block
 * @vsig: the VSIG to which this profile is to be added
 * @hdl: the profile handle indicating the profile to add
 * @rev: true to add entries to the end of the list
 * @chg: the change list
 */
enum ice_status
ice_add_prof_id_vsig(struct ice_hw *hw, enum ice_block blk, uint16_t vsig,
    uint64_t hdl, bool rev, struct ice_chs_chg_head *chg)
{
	/* Masks that ignore flags */
	uint8_t vl_msk[ICE_TCAM_KEY_VAL_SZ] = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
	uint8_t dc_msk[ICE_TCAM_KEY_VAL_SZ] = { 0xFF, 0xFF, 0x00, 0x00, 0x00 };
	uint8_t nm_msk[ICE_TCAM_KEY_VAL_SZ] = { 0x00, 0x00, 0x00, 0x00, 0x00 };
	enum ice_status status = ICE_SUCCESS;
	struct ice_prof_map *map;
	struct ice_vsig_prof *t;
	struct ice_chs_chg *p;
	uint16_t vsig_idx, i;

	/* Error, if this VSIG already has this profile */
	if (ice_has_prof_vsig(hw, blk, vsig, hdl))
		return ICE_ERR_ALREADY_EXISTS;

	/* new VSIG profile structure */
	t = (struct ice_vsig_prof *)ice_malloc(hw, sizeof(*t));
	if (!t)
		return ICE_ERR_NO_MEMORY;
#if 0
	ice_acquire_lock(&hw->blk[blk].es.prof_map_lock);
#endif
	/* Get the details on the profile specified by the handle ID */
	map = ice_search_prof_id(hw, blk, hdl);
	if (!map) {
		status = ICE_ERR_DOES_NOT_EXIST;
		goto err_ice_add_prof_id_vsig;
	}

	t->profile_cookie = map->profile_cookie;
	t->prof_id = map->prof_id;
	t->tcam_count = map->ptg_cnt;

	/* create TCAM entries */
	for (i = 0; i < map->ptg_cnt; i++) {
		uint16_t tcam_idx;

		/* add TCAM to change list */
		p = (struct ice_chs_chg *)ice_malloc(hw, sizeof(*p));
		if (!p) {
			status = ICE_ERR_NO_MEMORY;
			goto err_ice_add_prof_id_vsig;
		}

		/* allocate the TCAM entry index */
		status = ice_alloc_tcam_ent(hw, blk, true, &tcam_idx);
		if (status) {
			ice_free(hw, p);
			goto err_ice_add_prof_id_vsig;
		}

		t->tcam[i].ptg = map->ptg[i];
		t->tcam[i].prof_id = map->prof_id;
		t->tcam[i].tcam_idx = tcam_idx;
		t->tcam[i].in_use = true;

		p->type = ICE_TCAM_ADD;
		p->add_tcam_idx = true;
		p->prof_id = t->tcam[i].prof_id;
		p->ptg = t->tcam[i].ptg;
		p->vsig = vsig;
		p->tcam_idx = t->tcam[i].tcam_idx;

		/* write the TCAM entry */
		status = ice_tcam_write_entry(hw, blk, t->tcam[i].tcam_idx,
					      t->tcam[i].prof_id,
					      t->tcam[i].ptg, vsig, 0, 0,
					      vl_msk, dc_msk, nm_msk);
		if (status) {
			ice_free(hw, p);
			goto err_ice_add_prof_id_vsig;
		}

		/* log change */
		TAILQ_INSERT_HEAD(chg, p, list_entry);
	}

	/* add profile to VSIG */
	vsig_idx = vsig & ICE_VSIG_IDX_M;
	if (rev)
		TAILQ_INSERT_TAIL(
		    &hw->blk[blk].xlt2.vsig_tbl[vsig_idx].prop_lst, t, list);
	else
		TAILQ_INSERT_HEAD(
		    &hw->blk[blk].xlt2.vsig_tbl[vsig_idx].prop_lst, t, list);
#if 0
	ice_release_lock(&hw->blk[blk].es.prof_map_lock);
#endif
	return status;

err_ice_add_prof_id_vsig:
#if 0
	ice_release_lock(&hw->blk[blk].es.prof_map_lock);
#endif
	/* let caller clean up the change list */
	ice_free(hw, t);
	return status;
}

/**
 * ice_create_vsig_from_lst - create a new VSIG with a list of profiles
 * @hw: pointer to the HW struct
 * @blk: hardware block
 * @vsi: the initial VSI that will be in VSIG
 * @lst: the list of profile that will be added to the VSIG
 * @new_vsig: return of new VSIG
 * @chg: the change list
 */
enum ice_status
ice_create_vsig_from_lst(struct ice_hw *hw, enum ice_block blk, uint16_t vsi,
			 struct ice_vsig_prof_head *lst, uint16_t *new_vsig,
			 struct ice_chs_chg_head *chg)
{
	struct ice_vsig_prof *t;
	enum ice_status status;
	uint16_t vsig;

	vsig = ice_vsig_alloc(hw, blk);
	if (!vsig)
		return ICE_ERR_HW_TABLE;

	status = ice_move_vsi(hw, blk, vsi, vsig, chg);
	if (status)
		return status;

	TAILQ_FOREACH(t, lst, list) {
		/* Reverse the order here since we are copying the list */
		status = ice_add_prof_id_vsig(hw, blk, vsig, t->profile_cookie,
					      true, chg);
		if (status)
			return status;
	}

	*new_vsig = vsig;

	return ICE_SUCCESS;
}

/**
 * ice_upd_prof_hw - update hardware using the change list
 * @hw: pointer to the HW struct
 * @blk: hardware block
 * @chgs: the list of changes to make in hardware
 */
enum ice_status
ice_upd_prof_hw(struct ice_hw *hw, enum ice_block blk,
		struct ice_chs_chg_head *chgs)
{
#if 0
	struct ice_buf_build *b;
	struct ice_chs_chg *tmp;
	enum ice_status status;
	uint16_t pkg_sects;
	uint16_t xlt1 = 0;
	uint16_t xlt2 = 0;
	uint16_t tcam = 0;
	uint16_t es = 0;
	uint16_t sects;

	/* count number of sections we need */
	TAILQ_FOREACH(p, chgs, list_entry) {
		switch (tmp->type) {
		case ICE_PTG_ES_ADD:
			if (tmp->add_ptg)
				xlt1++;
			if (tmp->add_prof)
				es++;
			break;
		case ICE_TCAM_ADD:
			tcam++;
			break;
		case ICE_VSIG_ADD:
		case ICE_VSI_MOVE:
		case ICE_VSIG_REM:
			xlt2++;
			break;
		default:
			break;
		}
	}
	sects = xlt1 + xlt2 + tcam + es;

	if (!sects)
		return ICE_SUCCESS;

	/* Build update package buffer */
	b = ice_pkg_buf_alloc(hw);
	if (!b)
		return ICE_ERR_NO_MEMORY;

	status = ice_pkg_buf_reserve_section(b, sects);
	if (status)
		goto error_tmp;

	/* Preserve order of table update: ES, TCAM, PTG, VSIG */
	if (es) {
		status = ice_prof_bld_es(hw, blk, b, chgs);
		if (status)
			goto error_tmp;
	}

	if (tcam) {
		status = ice_prof_bld_tcam(hw, blk, b, chgs);
		if (status)
			goto error_tmp;
	}

	if (xlt1) {
		status = ice_prof_bld_xlt1(blk, b, chgs);
		if (status)
			goto error_tmp;
	}

	if (xlt2) {
		status = ice_prof_bld_xlt2(blk, b, chgs);
		if (status)
			goto error_tmp;
	}

	/* After package buffer build check if the section count in buffer is
	 * non-zero and matches the number of sections detected for package
	 * update.
	 */
	pkg_sects = ice_pkg_buf_get_active_sections(b);
	if (!pkg_sects || pkg_sects != sects) {
		status = ICE_ERR_INVAL_SIZE;
		goto error_tmp;
	}

	/* update package */
	status = ice_update_pkg(hw, ice_pkg_buf(b), 1);
	if (status == ICE_ERR_AQ_ERROR)
		ice_debug(hw, ICE_DBG_INIT, "Unable to update HW profile\n");

error_tmp:
	ice_pkg_buf_free(hw, b);
	return status;
#else
	printf("%s: not implemented\n", __func__);
	return ICE_ERR_NOT_IMPL;
#endif
}

/**
 * ice_match_prop_lst - determine if properties of two lists match
 * @list1: first properties list
 * @list2: second properties list
 *
 * Count, cookies and the order must match in order to be considered equivalent.
 */
bool
ice_match_prop_lst(struct ice_vsig_prof_head *list1,
    struct ice_vsig_prof_head *list2)
{
	struct ice_vsig_prof *tmp1, *tmp2;
	uint16_t chk_count = 0;
	uint16_t count = 0;

	/* compare counts */
	TAILQ_FOREACH(tmp1, list1, list)
		count++;
	TAILQ_FOREACH(tmp2, list2, list)
		chk_count++;
	if (!count || count != chk_count)
		return false;

	tmp1 = TAILQ_FIRST(list1);
	tmp2 = TAILQ_FIRST(list2);

	/* profile cookies must compare, and in the exact same order to take
	 * into account priority
	 */
	while (count--) {
		if (tmp2->profile_cookie != tmp1->profile_cookie)
			return false;

		tmp1 = TAILQ_NEXT(tmp1, list);
		tmp2 = TAILQ_NEXT(tmp2, list);
	}

	return true;
}
/**
 * ice_find_dup_props_vsig - find VSI group with a specified set of properties
 * @hw: pointer to the hardware structure
 * @blk: HW block
 * @chs: characteristic list
 * @vsig: returns the VSIG with the matching profiles, if found
 *
 * Each VSIG is associated with a characteristic set; i.e. all VSIs under
 * a group have the same characteristic set. To check if there exists a VSIG
 * which has the same characteristics as the input characteristics; this
 * function will iterate through the XLT2 list and return the VSIG that has a
 * matching configuration. In order to make sure that priorities are accounted
 * for, the list must match exactly, including the order in which the
 * characteristics are listed.
 */
enum ice_status
ice_find_dup_props_vsig(struct ice_hw *hw, enum ice_block blk,
			struct ice_vsig_prof_head *lst, uint16_t *vsig)
{
	struct ice_xlt2 *xlt2 = &hw->blk[blk].xlt2;
	uint16_t i;

	for (i = 0; i < xlt2->count; i++) {
		if (xlt2->vsig_tbl[i].in_use &&
		    ice_match_prop_lst(lst, &xlt2->vsig_tbl[i].prop_lst)) {
			*vsig = ICE_VSIG_VALUE(i, hw->pf_id);
			return ICE_SUCCESS;
		}
	}
	return ICE_ERR_DOES_NOT_EXIST;
}

/**
 * ice_rem_prof_id_flow - remove flow
 * @hw: pointer to the HW struct
 * @blk: hardware block
 * @vsi: the VSI from which to remove the profile specified by ID
 * @hdl: profile tracking handle
 *
 * Calling this function will update the hardware tables to remove the
 * profile indicated by the ID parameter for the VSIs specified in the VSI
 * array. Once successfully called, the flow will be disabled.
 */
enum ice_status
ice_rem_prof_id_flow(struct ice_hw *hw, enum ice_block blk, uint16_t vsi,
    uint64_t hdl)
{
	struct ice_vsig_prof *tmp1, *del1;
	struct ice_chs_chg *tmp, *del;
	struct ice_chs_chg_head chg;
	struct ice_vsig_prof_head copy;
	enum ice_status status;
	uint16_t vsig;

	TAILQ_INIT(&copy);
	TAILQ_INIT(&chg);

	/* determine if VSI is already part of a VSIG */
	status = ice_vsig_find_vsi(hw, blk, vsi, &vsig);
	if (!status && vsig) {
		bool last_profile;
		bool only_vsi;
		uint16_t ref;

		/* found in VSIG */
		last_profile = ice_vsig_prof_id_count(hw, blk, vsig) == 1;
		status = ice_vsig_get_ref(hw, blk, vsig, &ref);
		if (status)
			goto err_ice_rem_prof_id_flow;
		only_vsi = (ref == 1);

		if (only_vsi) {
			/* If the original VSIG only contains one reference,
			 * which will be the requesting VSI, then the VSI is not
			 * sharing entries and we can simply remove the specific
			 * characteristics from the VSIG.
			 */

			if (last_profile) {
				/* If there are no profiles left for this VSIG,
				 * then simply remove the VSIG.
				 */
				status = ice_rem_vsig(hw, blk, vsig, &chg);
				if (status)
					goto err_ice_rem_prof_id_flow;
			} else {
				status = ice_rem_prof_id_vsig(hw, blk, vsig,
							      hdl, &chg);
				if (status)
					goto err_ice_rem_prof_id_flow;

				/* Adjust priorities */
				status = ice_adj_prof_priorities(hw, blk, vsig,
								 &chg);
				if (status)
					goto err_ice_rem_prof_id_flow;
			}

		} else {
			/* Make a copy of the VSIG's list of Profiles */
			status = ice_get_profs_vsig(hw, blk, vsig, &copy);
			if (status)
				goto err_ice_rem_prof_id_flow;

			/* Remove specified profile entry from the list */
			status = ice_rem_prof_from_list(hw, &copy, hdl);
			if (status)
				goto err_ice_rem_prof_id_flow;

			if (TAILQ_EMPTY(&copy)) {
				status = ice_move_vsi(hw, blk, vsi,
						      ICE_DEFAULT_VSIG, &chg);
				if (status)
					goto err_ice_rem_prof_id_flow;

			} else if (!ice_find_dup_props_vsig(hw, blk, &copy,
							    &vsig)) {
				/* found an exact match */
				/* add or move VSI to the VSIG that matches */
				/* Search for a VSIG with a matching profile
				 * list
				 */

				/* Found match, move VSI to the matching VSIG */
				status = ice_move_vsi(hw, blk, vsi, vsig, &chg);
				if (status)
					goto err_ice_rem_prof_id_flow;
			} else {
				/* since no existing VSIG supports this
				 * characteristic pattern, we need to create a
				 * new VSIG and TCAM entries
				 */
				status = ice_create_vsig_from_lst(hw, blk, vsi,
								  &copy, &vsig,
								  &chg);
				if (status)
					goto err_ice_rem_prof_id_flow;

				/* Adjust priorities */
				status = ice_adj_prof_priorities(hw, blk, vsig,
								 &chg);
				if (status)
					goto err_ice_rem_prof_id_flow;
			}
		}
	} else {
		status = ICE_ERR_DOES_NOT_EXIST;
	}

	/* update hardware tables */
	if (!status)
		status = ice_upd_prof_hw(hw, blk, &chg);

err_ice_rem_prof_id_flow:
	TAILQ_FOREACH_SAFE(del, &chg, list_entry, tmp) {
		TAILQ_REMOVE(&chg, del, list_entry);
		ice_free(hw, del);
	}

	TAILQ_FOREACH_SAFE(del1, &copy, list, tmp1) {
		TAILQ_REMOVE(&copy, del1, list);
		ice_free(hw, del1);
	}

	return status;
}

/**
 * ice_flow_disassoc_prof - disassociate a VSI from a flow profile
 * @hw: pointer to the hardware structure
 * @blk: classification stage
 * @prof: pointer to flow profile
 * @vsi_handle: software VSI handle
 *
 * Assumption: the caller has acquired the lock to the profile list
 * and the software VSI handle has been validated
 */
enum ice_status
ice_flow_disassoc_prof(struct ice_hw *hw, enum ice_block blk,
		       struct ice_flow_prof *prof, uint16_t vsi_handle)
{
	enum ice_status status = ICE_SUCCESS;

	if (ice_is_bit_set(prof->vsis, vsi_handle)) {
		status = ice_rem_prof_id_flow(hw, blk,
		    hw->vsi_ctx[vsi_handle]->vsi_num, prof->id);
		if (!status)
			ice_clear_bit(vsi_handle, prof->vsis);
		else
			DNPRINTF(ICE_DBG_FLOW,
			    "%s: HW profile remove failed, %d\n",
			    __func__, status);
	}

	return status;
}

/**
 * ice_flow_rem_prof - Remove a flow profile and all entries associated with it
 * @hw: pointer to the HW struct
 * @blk: the block for which the flow profile is to be removed
 * @prof_id: unique ID of the flow profile to be removed
 */
enum ice_status
ice_flow_rem_prof(struct ice_hw *hw, enum ice_block blk, uint64_t prof_id)
{
#if 0
	struct ice_flow_prof *prof;
	enum ice_status status;

	ice_acquire_lock(&hw->fl_profs_locks[blk]);

	prof = ice_flow_find_prof_id(hw, blk, prof_id);
	if (!prof) {
		status = ICE_ERR_DOES_NOT_EXIST;
		goto out;
	}

	/* prof becomes invalid after the call */
	status = ice_flow_rem_prof_sync(hw, blk, prof);

out:
	ice_release_lock(&hw->fl_profs_locks[blk]);

	return status;
#else
	printf("%s: not implemented", __func__);
	return ICE_ERR_NOT_IMPL;
#endif
}
/**
 * ice_rem_vsi_rss_cfg - remove RSS configurations associated with VSI
 * @hw: pointer to the hardware structure
 * @vsi_handle: software VSI handle
 *
 * This function will iterate through all flow profiles and disassociate
 * the VSI from that profile. If the flow profile has no VSIs it will
 * be removed.
 */
enum ice_status
ice_rem_vsi_rss_cfg(struct ice_hw *hw, uint16_t vsi_handle)
{
	const enum ice_block blk = ICE_BLK_RSS;
	struct ice_flow_prof *p, *t;
	enum ice_status status = ICE_SUCCESS;
	uint16_t vsig;

	if (!ice_is_vsi_valid(hw, vsi_handle))
		return ICE_ERR_PARAM;

	if (TAILQ_EMPTY(&hw->fl_profs[blk]))
		return ICE_SUCCESS;
#if 0
	ice_acquire_lock(&hw->rss_locks);
#endif
	TAILQ_FOREACH_SAFE(p, &hw->fl_profs[blk], l_entry, t) {
		int ret;

		/* check if vsig is already removed */
		ret = ice_vsig_find_vsi(hw, blk,
		    hw->vsi_ctx[vsi_handle]->vsi_num, &vsig);
		if (!ret && !vsig)
			break;

		if (ice_is_bit_set(p->vsis, vsi_handle)) {
			status = ice_flow_disassoc_prof(hw, blk, p, vsi_handle);
			if (status)
				break;

			if (!ice_is_any_bit_set(p->vsis, ICE_MAX_VSI)) {
				status = ice_flow_rem_prof(hw, blk, p->id);
				if (status)
					break;
			}
		}
	}
#if 0
	ice_release_lock(&hw->rss_locks);
#endif
	return status;
}

/**
 * ice_rem_vsi_rss_list - remove VSI from RSS list
 * @hw: pointer to the hardware structure
 * @vsi_handle: software VSI handle
 *
 * Remove the VSI from all RSS configurations in the list.
 */
void
ice_rem_vsi_rss_list(struct ice_hw *hw, uint16_t vsi_handle)
{
#if 0
	struct ice_rss_cfg *r, *tmp;

	if (LIST_EMPTY(&hw->rss_list_head))
		return;

	ice_acquire_lock(&hw->rss_locks);
	LIST_FOR_EACH_ENTRY_SAFE(r, tmp, &hw->rss_list_head,
				 ice_rss_cfg, l_entry)
		if (ice_test_and_clear_bit(vsi_handle, r->vsis))
			if (!ice_is_any_bit_set(r->vsis, ICE_MAX_VSI)) {
				LIST_DEL(&r->l_entry);
				ice_free(hw, r);
			}
	ice_release_lock(&hw->rss_locks);
#else
	printf("%s: not implemented\n", __func__);
#endif
}

/**
 * ice_clean_vsi_rss_cfg - Cleanup RSS configuration for a given VSI
 * @vsi: pointer to the VSI structure
 *
 * Cleanup the advanced RSS configuration for a given VSI. This is necessary
 * during driver removal to ensure that all RSS resources are properly
 * released.
 *
 * @remark this function doesn't report an error as it is expected to be
 * called during driver reset and unload, and there isn't much the driver can
 * do if freeing RSS resources fails.
 */
void
ice_clean_vsi_rss_cfg(struct ice_vsi *vsi)
{
	struct ice_softc *sc = vsi->sc;
	struct ice_hw *hw = &sc->hw;
	enum ice_status status;

	status = ice_rem_vsi_rss_cfg(hw, vsi->idx);
	if (status)
		printf("%s: Failed to remove RSS configuration for VSI %d, "
		    "err %s\n", sc->sc_dev.dv_xname, vsi->idx,
		    ice_status_str(status));

	/* Remove this VSI from the RSS list */
	ice_rem_vsi_rss_list(hw, vsi->idx);
}

/**
 * ice_remove_vsi_mirroring -- Teardown any VSI mirroring rules
 * @vsi: VSI to remove mirror rules from
 */
void
ice_remove_vsi_mirroring(struct ice_vsi *vsi)
{
#if 0
	struct ice_hw *hw = &vsi->sc->hw;
	enum ice_status status = ICE_SUCCESS;
	bool keep_alloc = false;

	if (vsi->rule_mir_ingress != ICE_INVAL_MIRROR_RULE_ID)
		status = ice_aq_delete_mir_rule(hw, vsi->rule_mir_ingress, keep_alloc, NULL);

	if (status)
		device_printf(vsi->sc->dev, "Could not remove mirror VSI ingress rule, err %s aq_err %s\n",
			      ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status));

	status = ICE_SUCCESS;

	if (vsi->rule_mir_egress != ICE_INVAL_MIRROR_RULE_ID)
		status = ice_aq_delete_mir_rule(hw, vsi->rule_mir_egress, keep_alloc, NULL);

	if (status)
		device_printf(vsi->sc->dev, "Could not remove mirror VSI egress rule, err %s aq_err %s\n",
			      ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status));
#else
	printf("%s: not implemented\n", __func__);
#endif
}

/**
 * ice_sched_get_vsi_node - Get a VSI node based on VSI ID
 * @pi: pointer to the port information structure
 * @tc_node: pointer to the TC node
 * @vsi_handle: software VSI handle
 *
 * This function retrieves a VSI node for a given VSI ID from a given
 * TC branch
 */
struct ice_sched_node *
ice_sched_get_vsi_node(struct ice_port_info *pi, struct ice_sched_node *tc_node,
		       uint16_t vsi_handle)
{
	struct ice_sched_node *node;
	uint8_t vsi_layer;

	vsi_layer = ice_sched_get_vsi_layer(pi->hw);
	node = ice_sched_get_first_node(pi, tc_node, vsi_layer);

	/* Check whether it already exists */
	while (node) {
		if (node->vsi_handle == vsi_handle)
			return node;
		node = node->sibling;
	}

	return node;
}

/**
 * ice_sched_is_leaf_node_present - check for a leaf node in the sub-tree
 * @node: pointer to the sub-tree node
 *
 * This function checks for a leaf node presence in a given sub-tree node.
 */
bool
ice_sched_is_leaf_node_present(struct ice_sched_node *node)
{
	uint8_t i;

	for (i = 0; i < node->num_children; i++)
		if (ice_sched_is_leaf_node_present(node->children[i]))
			return true;
	/* check for a leaf node */
	return (node->info.data.elem_type == ICE_AQC_ELEM_TYPE_LEAF);
}

/**
 * ice_sched_rm_agg_vsi_info - remove aggregator related VSI info entry
 * @pi: port information structure
 * @vsi_handle: software VSI handle
 *
 * This function removes single aggregator VSI info entry from
 * aggregator list.
 */
void
ice_sched_rm_agg_vsi_info(struct ice_port_info *pi, uint16_t vsi_handle)
{
#if 0
	struct ice_sched_agg_info *agg_info;
	struct ice_sched_agg_info *atmp;

	LIST_FOR_EACH_ENTRY_SAFE(agg_info, atmp, &pi->hw->agg_list,
				 ice_sched_agg_info,
				 list_entry) {
		struct ice_sched_agg_vsi_info *agg_vsi_info;
		struct ice_sched_agg_vsi_info *vtmp;

		LIST_FOR_EACH_ENTRY_SAFE(agg_vsi_info, vtmp,
					 &agg_info->agg_vsi_list,
					 ice_sched_agg_vsi_info, list_entry)
			if (agg_vsi_info->vsi_handle == vsi_handle) {
				LIST_DEL(&agg_vsi_info->list_entry);
				ice_free(pi->hw, agg_vsi_info);
				return;
			}
	}
#else
	printf("%s: not implemented\n", __func__);
#endif
}

/**
 * ice_sched_rm_vsi_cfg - remove the VSI and its children nodes
 * @pi: port information structure
 * @vsi_handle: software VSI handle
 * @owner: LAN or RDMA
 *
 * This function removes the VSI and its LAN or RDMA children nodes from the
 * scheduler tree.
 */
enum ice_status
ice_sched_rm_vsi_cfg(struct ice_port_info *pi, uint16_t vsi_handle,
    uint8_t owner)
{
	enum ice_status status = ICE_ERR_PARAM;
	struct ice_vsi_ctx *vsi_ctx;
	uint8_t i;

	DNPRINTF(ICE_DBG_SCHED, "%s: removing VSI %d\n", __func__, vsi_handle);
	if (!ice_is_vsi_valid(pi->hw, vsi_handle))
		return status;
#if 0
	ice_acquire_lock(&pi->sched_lock);
#endif
	vsi_ctx = ice_get_vsi_ctx(pi->hw, vsi_handle);
	if (!vsi_ctx)
		goto exit_sched_rm_vsi_cfg;

	ice_for_each_traffic_class(i) {
		struct ice_sched_node *vsi_node, *tc_node;
		uint8_t j = 0;

		tc_node = ice_sched_get_tc_node(pi, i);
		if (!tc_node)
			continue;

		vsi_node = ice_sched_get_vsi_node(pi, tc_node, vsi_handle);
		if (!vsi_node)
			continue;

		if (ice_sched_is_leaf_node_present(vsi_node)) {
			DNPRINTF(ICE_DBG_SCHED,
			    "%s: VSI has leaf nodes in TC %d\n", __func__, i);
			status = ICE_ERR_IN_USE;
			goto exit_sched_rm_vsi_cfg;
		}
		while (j < vsi_node->num_children) {
			if (vsi_node->children[j]->owner == owner) {
				ice_free_sched_node(pi, vsi_node->children[j]);

				/* reset the counter again since the num
				 * children will be updated after node removal
				 */
				j = 0;
			} else {
				j++;
			}
		}
		/* remove the VSI if it has no children */
		if (!vsi_node->num_children) {
			ice_free_sched_node(pi, vsi_node);
			vsi_ctx->sched.vsi_node[i] = NULL;

			/* clean up aggregator related VSI info if any */
			ice_sched_rm_agg_vsi_info(pi, vsi_handle);
		}
		if (owner == ICE_SCHED_NODE_OWNER_LAN)
			vsi_ctx->sched.max_lanq[i] = 0;
		else
			vsi_ctx->sched.max_rdmaq[i] = 0;
	}
	status = ICE_SUCCESS;

exit_sched_rm_vsi_cfg:
#if 0
	ice_release_lock(&pi->sched_lock);
#endif
	return status;
}

/**
 * ice_rm_vsi_lan_cfg - remove VSI and its LAN children nodes
 * @pi: port information structure
 * @vsi_handle: software VSI handle
 *
 * This function clears the VSI and its LAN children nodes from scheduler tree
 * for all TCs.
 */
enum ice_status
ice_rm_vsi_lan_cfg(struct ice_port_info *pi, uint16_t vsi_handle)
{
	return ice_sched_rm_vsi_cfg(pi, vsi_handle, ICE_SCHED_NODE_OWNER_LAN);
}

/**
 * ice_aq_free_vsi
 * @hw: pointer to the HW struct
 * @vsi_ctx: pointer to a VSI context struct
 * @keep_vsi_alloc: keep VSI allocation as part of this PF's resources
 * @cd: pointer to command details structure or NULL
 *
 * Free VSI context info from hardware (0x0213)
 */
enum ice_status
ice_aq_free_vsi(struct ice_hw *hw, struct ice_vsi_ctx *vsi_ctx,
		bool keep_vsi_alloc, struct ice_sq_cd *cd)
{
	struct ice_aqc_add_update_free_vsi_resp *resp;
	struct ice_aqc_add_get_update_free_vsi *cmd;
	struct ice_aq_desc desc;
	enum ice_status status;

	cmd = &desc.params.vsi_cmd;
	resp = &desc.params.add_update_free_vsi_res;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_free_vsi);

	cmd->vsi_num = htole16(vsi_ctx->vsi_num | ICE_AQ_VSI_IS_VALID);
	if (keep_vsi_alloc)
		cmd->cmd_flags = htole16(ICE_AQ_VSI_KEEP_ALLOC);

	status = ice_aq_send_cmd(hw, &desc, NULL, 0, cd);
	if (!status) {
		vsi_ctx->vsis_allocd = le16toh(resp->vsi_used);
		vsi_ctx->vsis_unallocated = le16toh(resp->vsi_free);
	}

	return status;
}

/**
 * ice_clear_vsi_q_ctx - clear VSI queue contexts for all TCs
 * @hw: pointer to the HW struct
 * @vsi_handle: VSI handle
 */
void
ice_clear_vsi_q_ctx(struct ice_hw *hw, uint16_t vsi_handle)
{
	struct ice_vsi_ctx *vsi;
	uint8_t i;

	vsi = ice_get_vsi_ctx(hw, vsi_handle);
	if (!vsi)
		return;
	ice_for_each_traffic_class(i) {
		if (vsi->lan_q_ctx[i]) {
			ice_free(hw, vsi->lan_q_ctx[i]);
			vsi->lan_q_ctx[i] = NULL;
		}
		if (vsi->rdma_q_ctx[i]) {
			ice_free(hw, vsi->rdma_q_ctx[i]);
			vsi->rdma_q_ctx[i] = NULL;
		}
	}
}

/**
 * ice_clear_vsi_ctx - clear the VSI context entry
 * @hw: pointer to the HW struct
 * @vsi_handle: VSI handle
 *
 * clear the VSI context entry
 */
void
ice_clear_vsi_ctx(struct ice_hw *hw, uint16_t vsi_handle)
{
	struct ice_vsi_ctx *vsi;

	vsi = ice_get_vsi_ctx(hw, vsi_handle);
	if (vsi) {
		ice_clear_vsi_q_ctx(hw, vsi_handle);
		ice_free(hw, vsi);
		hw->vsi_ctx[vsi_handle] = NULL;
	}
}

/**
 * ice_free_vsi- free VSI context from hardware and VSI handle list
 * @hw: pointer to the HW struct
 * @vsi_handle: unique VSI handle
 * @vsi_ctx: pointer to a VSI context struct
 * @keep_vsi_alloc: keep VSI allocation as part of this PF's resources
 * @cd: pointer to command details structure or NULL
 *
 * Free VSI context info from hardware as well as from VSI handle list
 */
enum ice_status
ice_free_vsi(struct ice_hw *hw, uint16_t vsi_handle,
    struct ice_vsi_ctx *vsi_ctx, bool keep_vsi_alloc, struct ice_sq_cd *cd)
{
	enum ice_status status;

	if (!ice_is_vsi_valid(hw, vsi_handle))
		return ICE_ERR_PARAM;
	vsi_ctx->vsi_num = hw->vsi_ctx[vsi_handle]->vsi_num;
	status = ice_aq_free_vsi(hw, vsi_ctx, keep_vsi_alloc, cd);
	if (!status)
		ice_clear_vsi_ctx(hw, vsi_handle);
	return status;
}

/**
 * ice_deinit_vsi - Tell firmware to release resources for a VSI
 * @vsi: the VSI to release
 *
 * Helper function which requests the firmware to release the hardware
 * resources associated with a given VSI.
 */
void
ice_deinit_vsi(struct ice_vsi *vsi)
{
	struct ice_vsi_ctx ctx = { 0 };
	struct ice_softc *sc = vsi->sc;
	struct ice_hw *hw = &sc->hw;
	enum ice_status status;

	/* Assert that the VSI pointer matches in the list */
	KASSERT(vsi == sc->all_vsi[vsi->idx]);

	ctx.info = vsi->info;

	status = ice_rm_vsi_lan_cfg(hw->port_info, vsi->idx);
	if (status) {
		/*
		 * This should only fail if the VSI handle is invalid, or if
		 * any of the nodes have leaf nodes which are still in use.
		 */
		printf("%s: Unable to remove scheduler nodes for VSI %d, "
		    "err %s\n", sc->sc_dev.dv_xname, vsi->idx,
		    ice_status_str(status));
	}

	/* Tell firmware to release the VSI resources */
	status = ice_free_vsi(hw, vsi->idx, &ctx, false, NULL);
	if (status != 0) {
		printf("%s: Free VSI %u AQ call failed, err %s aq_err %s\n",
		    sc->sc_dev.dv_xname, vsi->idx, ice_status_str(status),
		    ice_aq_str(hw->adminq.sq_last_status));
	}
}

/**
 * ice_free_vsi_qmaps - Free the PF qmaps associated with a VSI
 * @vsi: the VSI private structure
 *
 * Frees the PF qmaps associated with the given VSI. Generally this will be
 * called by ice_release_vsi, but may need to be called during attach cleanup,
 * depending on when the qmaps were allocated.
 */
void
ice_free_vsi_qmaps(struct ice_vsi *vsi)
{
	struct ice_softc *sc = vsi->sc;

	if (vsi->tx_qmap) {
		ice_resmgr_release_map(&sc->tx_qmgr, vsi->tx_qmap,
		    vsi->num_tx_queues);
		free(vsi->tx_qmap, M_DEVBUF,
		    vsi->num_tx_queues * sizeof(uint16_t));
		vsi->tx_qmap = NULL;
	}

	if (vsi->rx_qmap) {
		ice_resmgr_release_map(&sc->rx_qmgr, vsi->rx_qmap,
		     vsi->num_rx_queues);
		free(vsi->rx_qmap, M_DEVBUF,
		    vsi->num_rx_queues * sizeof(uint16_t));
		vsi->rx_qmap = NULL;
	}
}

/**
 * ice_release_vsi - Release resources associated with a VSI
 * @vsi: the VSI to release
 *
 * Release software and firmware resources associated with a VSI. Release the
 * queue managers associated with this VSI. Also free the VSI structure memory
 * if the VSI was allocated dynamically using ice_alloc_vsi().
 */
void
ice_release_vsi(struct ice_vsi *vsi)
{
	struct ice_softc *sc = vsi->sc;
	int idx = vsi->idx;

	/* Assert that the VSI pointer matches in the list */
	KASSERT(vsi == sc->all_vsi[idx]);

	/* Cleanup RSS configuration */
	if (ice_is_bit_set(sc->feat_en, ICE_FEATURE_RSS))
		ice_clean_vsi_rss_cfg(vsi);
#if 0
	ice_del_vsi_sysctl_ctx(vsi);
#endif
	/* Remove the configured mirror rule, if it exists */
	ice_remove_vsi_mirroring(vsi);

	/*
	 * If we unload the driver after a reset fails, we do not need to do
	 * this step.
	 */
	if (!ice_test_state(&sc->state, ICE_STATE_RESET_FAILED))
		ice_deinit_vsi(vsi);

	ice_free_vsi_qmaps(vsi);

	if (vsi->dynamic)
		free(sc->all_vsi[idx], M_DEVBUF, sizeof(struct ice_vsi));

	sc->all_vsi[idx] = NULL;
}

/**
 * ice_transition_recovery_mode - Transition to recovery mode
 * @sc: the device private softc
 *
 * Called when the driver detects that the firmware has entered recovery mode
 * at run time.
 */
void
ice_transition_recovery_mode(struct ice_softc *sc)
{
#if 0
	struct ice_vsi *vsi = &sc->pf_vsi;
#endif
	struct ifnet *ifp = &sc->sc_ac.ac_if;
	int i;

	printf("%s: firmware has switched into recovery mode",
	    sc->sc_dev.dv_xname);

	/* Tell the stack that the link has gone down */
	ifp->if_link_state = LINK_STATE_DOWN;
	if_link_state_change(ifp);
#if 0
	ice_rdma_pf_detach(sc);
#endif
	ice_clear_bit(ICE_FEATURE_RDMA, sc->feat_cap);

	ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_en);
	ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_cap);
#if 0
	ice_vsi_del_txqs_ctx(vsi);
	ice_vsi_del_rxqs_ctx(vsi);
#endif
	for (i = 0; i < sc->num_available_vsi; i++) {
		if (sc->all_vsi[i])
			ice_release_vsi(sc->all_vsi[i]);
	}

	if (sc->all_vsi) {
		free(sc->all_vsi, M_DEVBUF,
		    sc->num_available_vsi * sizeof(struct ice_vsi *));
		sc->all_vsi = NULL;
	}
	sc->num_available_vsi = 0;

	/* Destroy the interrupt manager */
	ice_resmgr_destroy(&sc->dev_imgr);
	/* Destroy the queue managers */
	ice_resmgr_destroy(&sc->tx_qmgr);
	ice_resmgr_destroy(&sc->rx_qmgr);

	ice_deinit_hw(&sc->hw);
}

/**
 * ice_clear_hw_tbls - clear HW tables and flow profiles
 * @hw: pointer to the hardware structure
 */
void
ice_clear_hw_tbls(struct ice_hw *hw)
{
	uint8_t i;

	for (i = 0; i < ICE_BLK_COUNT; i++) {
		struct ice_prof_redir *prof_redir = &hw->blk[i].prof_redir;
		struct ice_prof_tcam *prof = &hw->blk[i].prof;
		struct ice_xlt1 *xlt1 = &hw->blk[i].xlt1;
		struct ice_xlt2 *xlt2 = &hw->blk[i].xlt2;
		struct ice_es *es = &hw->blk[i].es;

		if (hw->blk[i].is_list_init) {
			ice_free_prof_map(hw, i);
			ice_free_flow_profs(hw, i);
		}

		ice_free_vsig_tbl(hw, (enum ice_block)i);

		if (xlt1->ptypes)
			memset(xlt1->ptypes, 0,
			    xlt1->count * sizeof(*xlt1->ptypes));

		if (xlt1->ptg_tbl)
			memset(xlt1->ptg_tbl, 0,
			    ICE_MAX_PTGS * sizeof(*xlt1->ptg_tbl));

		if (xlt1->t)
			memset(xlt1->t, 0, xlt1->count * sizeof(*xlt1->t));

		if (xlt2->vsis)
			memset(xlt2->vsis, 0,
			    xlt2->count * sizeof(*xlt2->vsis));

		if (xlt2->vsig_tbl)
			memset(xlt2->vsig_tbl, 0,
			    xlt2->count * sizeof(*xlt2->vsig_tbl));

		if (xlt2->t)
			memset(xlt2->t, 0, xlt2->count * sizeof(*xlt2->t));

		if (prof->t)
			memset(prof->t, 0, prof->count * sizeof(*prof->t));

		if (prof_redir->t)
			memset(prof_redir->t, 0,
			    prof_redir->count * sizeof(*prof_redir->t));

		if (es->t)
			memset(es->t, 0, es->count * sizeof(*es->t) * es->fvw);

		if (es->ref_count)
			memset(es->ref_count, 0,
			    es->count * sizeof(*es->ref_count));

		if (es->written)
			memset(es->written, 0,
			    es->count * sizeof(*es->written));

	}
}

/**
 * ice_prepare_for_reset - Prepare device for an impending reset
 * @sc: The device private softc
 *
 * Prepare the driver for an impending reset, shutting down VSIs, clearing the
 * scheduler setup, and shutting down controlqs. Uses the
 * ICE_STATE_PREPARED_FOR_RESET to indicate whether we've already prepared the
 * driver for reset or not.
 */
void
ice_prepare_for_reset(struct ice_softc *sc)
{
	struct ice_hw *hw = &sc->hw;

	/* If we're already prepared, there's nothing to do */
	if (ice_testandset_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET))
		return;

	DPRINTF("%s: preparing to reset device\n", sc->sc_dev.dv_xname);

	/* In recovery mode, hardware is not initialized */
	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
		return;
#if 0
	/* inform the RDMA client */
	ice_rdma_notify_reset(sc);
	/* stop the RDMA client */
	ice_rdma_pf_stop(sc);
#endif
	/* Release the main PF VSI queue mappings */
	ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap,
				    sc->pf_vsi.num_tx_queues);
	ice_resmgr_release_map(&sc->rx_qmgr, sc->pf_vsi.rx_qmap,
				    sc->pf_vsi.num_rx_queues);
#if 0
	if (sc->mirr_if) {
		ice_resmgr_release_map(&sc->tx_qmgr, sc->mirr_if->vsi->tx_qmap,
		    sc->mirr_if->num_irq_vectors);
		ice_resmgr_release_map(&sc->rx_qmgr, sc->mirr_if->vsi->rx_qmap,
		    sc->mirr_if->num_irq_vectors);
	}
#endif
	ice_clear_hw_tbls(hw);

	if (hw->port_info)
		ice_sched_cleanup_all(hw);

	ice_shutdown_all_ctrlq(hw, false);
}

/**
 * ice_configure_misc_interrupts - enable 'other' interrupt causes
 * @sc: pointer to device private softc
 *
 * Enable various "other" interrupt causes, and associate them to interrupt 0,
 * which is our administrative interrupt.
 */
void
ice_configure_misc_interrupts(struct ice_softc *sc)
{
	struct ice_hw *hw = &sc->hw;
	uint32_t val;

	/* Read the OICR register to clear it */
	ICE_READ(hw, PFINT_OICR);

	/* Enable useful "other" interrupt causes */
	val = (PFINT_OICR_ECC_ERR_M |
	       PFINT_OICR_MAL_DETECT_M |
	       PFINT_OICR_GRST_M |
	       PFINT_OICR_PCI_EXCEPTION_M |
	       PFINT_OICR_VFLR_M |
	       PFINT_OICR_HMC_ERR_M |
	       PFINT_OICR_PE_CRITERR_M);

	ICE_WRITE(hw, PFINT_OICR_ENA, val);

	/* Note that since we're using MSI-X index 0, and ITR index 0, we do
	 * not explicitly program them when writing to the PFINT_*_CTL
	 * registers. Nevertheless, these writes are associating the
	 * interrupts with the ITR 0 vector
	 */

	/* Associate the OICR interrupt with ITR 0, and enable it */
	ICE_WRITE(hw, PFINT_OICR_CTL, PFINT_OICR_CTL_CAUSE_ENA_M);

	/* Associate the Mailbox interrupt with ITR 0, and enable it */
	ICE_WRITE(hw, PFINT_MBX_CTL, PFINT_MBX_CTL_CAUSE_ENA_M);

	/* Associate the AdminQ interrupt with ITR 0, and enable it */
	ICE_WRITE(hw, PFINT_FW_CTL, PFINT_FW_CTL_CAUSE_ENA_M);
}

void
ice_request_stack_reinit(struct ice_softc *sc)
{
	printf("%s: not implemented\n", __func__);
}

/**
 * ice_rebuild_recovery_mode - Rebuild driver state while in recovery mode
 * @sc: The device private softc
 *
 * Handle a driver rebuild while in recovery mode. This will only rebuild the
 * limited functionality supported while in recovery mode.
 */
void
ice_rebuild_recovery_mode(struct ice_softc *sc)
{
#if 0
	/* enable PCIe bus master */
	pci_enable_busmaster(dev);
#endif
	/* Configure interrupt causes for the administrative interrupt */
	ice_configure_misc_interrupts(sc);

	/* Enable ITR 0 right away, so that we can handle admin interrupts */
	ice_enable_intr(&sc->hw, 0);

	/* Rebuild is finished. We're no longer prepared to reset */
	ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET);

	printf("%s: device rebuild successful\n", sc->sc_dev.dv_xname);

	/* In order to completely restore device functionality, the iflib core
	 * needs to be reset. We need to request an iflib reset. Additionally,
	 * because the state of IFC_DO_RESET is cached within task_fn_admin in
	 * the iflib core, we also want re-run the admin task so that iflib
	 * resets immediately instead of waiting for the next interrupt.
	 */

	ice_request_stack_reinit(sc);
}

/**
 * ice_clean_all_vsi_rss_cfg - Cleanup RSS configuration for all VSIs
 * @sc: the device softc pointer
 *
 * Cleanup the advanced RSS configuration for all VSIs on a given PF
 * interface.
 *
 * @remark This should be called while preparing for a reset, to cleanup stale
 * RSS configuration for all VSIs.
 */
void
ice_clean_all_vsi_rss_cfg(struct ice_softc *sc)
{
	int i;

	/* No need to cleanup if RSS is not enabled */
	if (!ice_is_bit_set(sc->feat_en, ICE_FEATURE_RSS))
		return;

	for (i = 0; i < sc->num_available_vsi; i++) {
		struct ice_vsi *vsi = sc->all_vsi[i];

		if (vsi)
			ice_clean_vsi_rss_cfg(vsi);
	}
}

/**
 * ice_reset_pf_stats - Reset port stats counters
 * @sc: Device private softc structure
 *
 * Reset software tracking values for statistics to zero, and indicate that
 * offsets haven't been loaded. Intended to be called after a device reset so
 * that statistics count from zero again.
 */
void
ice_reset_pf_stats(struct ice_softc *sc)
{
#if 0
	memset(&sc->stats.prev, 0, sizeof(sc->stats.prev));
	memset(&sc->stats.cur, 0, sizeof(sc->stats.cur));
	sc->stats.offsets_loaded = false;
#endif
}

/**
 * ice_rebuild_pf_vsi_qmap - Rebuild the main PF VSI queue mapping
 * @sc: the device softc pointer
 *
 * Loops over the Tx and Rx queues for the main PF VSI and reassigns the queue
 * mapping after a reset occurred.
 */
int
ice_rebuild_pf_vsi_qmap(struct ice_softc *sc)
{
	struct ice_vsi *vsi = &sc->pf_vsi;
	struct ice_tx_queue *txq;
	struct ice_rx_queue *rxq;
	int err, i;

	/* Re-assign Tx queues from PF space to the main VSI */
	err = ice_resmgr_assign_contiguous(&sc->tx_qmgr, vsi->tx_qmap,
					    vsi->num_tx_queues);
	if (err) {
		printf("%s: Unable to re-assign PF Tx queues: %d\n",
		    sc->sc_dev.dv_xname, err);
		return (err);
	}

	/* Re-assign Rx queues from PF space to this VSI */
	err = ice_resmgr_assign_contiguous(&sc->rx_qmgr, vsi->rx_qmap,
					    vsi->num_rx_queues);
	if (err) {
		printf("%s: Unable to re-assign PF Rx queues: %d\n",
		    sc->sc_dev.dv_xname, err);
		goto err_release_tx_queues;
	}

	vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS;

	/* Re-assign Tx queue tail pointers */
	for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++)
		txq->tail = QTX_COMM_DBELL(vsi->tx_qmap[i]);

	/* Re-assign Rx queue tail pointers */
	for (i = 0, rxq = vsi->rx_queues; i < vsi->num_rx_queues; i++, rxq++)
		rxq->tail = QRX_TAIL(vsi->rx_qmap[i]);

	return (0);

err_release_tx_queues:
	ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap,
				   sc->pf_vsi.num_tx_queues);

	return (err);
}

#define ICE_UP_TABLE_TRANSLATE(val, i) \
		(((val) << ICE_AQ_VSI_UP_TABLE_UP##i##_S) & \
		ICE_AQ_VSI_UP_TABLE_UP##i##_M)

/**
 * ice_set_default_vsi_ctx - Setup default VSI context parameters
 * @ctx: the VSI context to initialize
 *
 * Initialize and prepare a default VSI context for configuring a new VSI.
 */
void
ice_set_default_vsi_ctx(struct ice_vsi_ctx *ctx)
{
	uint32_t table = 0;

	memset(&ctx->info, 0, sizeof(ctx->info));
	/* VSI will be allocated from shared pool */
	ctx->alloc_from_pool = true;
	/* Enable source pruning by default */
	ctx->info.sw_flags = ICE_AQ_VSI_SW_FLAG_SRC_PRUNE;
	/* Traffic from VSI can be sent to LAN */
	ctx->info.sw_flags2 = ICE_AQ_VSI_SW_FLAG_LAN_ENA;
	/* Allow all packets untagged/tagged */
	ctx->info.inner_vlan_flags = ((ICE_AQ_VSI_INNER_VLAN_TX_MODE_ALL &
				       ICE_AQ_VSI_INNER_VLAN_TX_MODE_M) >>
				       ICE_AQ_VSI_INNER_VLAN_TX_MODE_S);
	/* Show VLAN/UP from packets in Rx descriptors */
	ctx->info.inner_vlan_flags |= ((ICE_AQ_VSI_INNER_VLAN_EMODE_STR_BOTH &
					ICE_AQ_VSI_INNER_VLAN_EMODE_M) >>
					ICE_AQ_VSI_INNER_VLAN_EMODE_S);
	/* Have 1:1 UP mapping for both ingress/egress tables */
	table |= ICE_UP_TABLE_TRANSLATE(0, 0);
	table |= ICE_UP_TABLE_TRANSLATE(1, 1);
	table |= ICE_UP_TABLE_TRANSLATE(2, 2);
	table |= ICE_UP_TABLE_TRANSLATE(3, 3);
	table |= ICE_UP_TABLE_TRANSLATE(4, 4);
	table |= ICE_UP_TABLE_TRANSLATE(5, 5);
	table |= ICE_UP_TABLE_TRANSLATE(6, 6);
	table |= ICE_UP_TABLE_TRANSLATE(7, 7);
	ctx->info.ingress_table = htole32(table);
	ctx->info.egress_table = htole32(table);
	/* Have 1:1 UP mapping for outer to inner UP table */
	ctx->info.outer_up_table = htole32(table);
	/* No Outer tag support, so outer_vlan_flags remains zero */
}

/**
 * ice_set_rss_vsi_ctx - Setup VSI context parameters for RSS
 * @ctx: the VSI context to configure
 * @type: the VSI type
 *
 * Configures the VSI context for RSS, based on the VSI type.
 */
void
ice_set_rss_vsi_ctx(struct ice_vsi_ctx *ctx, enum ice_vsi_type type)
{
	uint8_t lut_type, hash_type;

	switch (type) {
	case ICE_VSI_PF:
		lut_type = ICE_AQ_VSI_Q_OPT_RSS_LUT_PF;
		hash_type = ICE_AQ_VSI_Q_OPT_RSS_TPLZ;
		break;
	case ICE_VSI_VF:
	case ICE_VSI_VMDQ2:
		lut_type = ICE_AQ_VSI_Q_OPT_RSS_LUT_VSI;
		hash_type = ICE_AQ_VSI_Q_OPT_RSS_TPLZ;
		break;
	default:
		/* Other VSI types do not support RSS */
		return;
	}

	ctx->info.q_opt_rss = (((lut_type << ICE_AQ_VSI_Q_OPT_RSS_LUT_S) &
				 ICE_AQ_VSI_Q_OPT_RSS_LUT_M) |
				((hash_type << ICE_AQ_VSI_Q_OPT_RSS_HASH_S) &
				 ICE_AQ_VSI_Q_OPT_RSS_HASH_M));
}

/**
 * ice_vsi_set_rss_params - Set the RSS parameters for the VSI
 * @vsi: the VSI to configure
 *
 * Sets the RSS table size and lookup table type for the VSI based on its
 * VSI type.
 */
void
ice_vsi_set_rss_params(struct ice_vsi *vsi)
{
	struct ice_softc *sc = vsi->sc;
	struct ice_hw_common_caps *cap;

	cap = &sc->hw.func_caps.common_cap;

	switch (vsi->type) {
	case ICE_VSI_PF:
		/* The PF VSI inherits RSS instance of the PF */
		vsi->rss_table_size = cap->rss_table_size;
		vsi->rss_lut_type = ICE_LUT_PF;
		break;
	case ICE_VSI_VF:
	case ICE_VSI_VMDQ2:
		vsi->rss_table_size = ICE_VSIQF_HLUT_ARRAY_SIZE;
		vsi->rss_lut_type = ICE_LUT_VSI;
		break;
	default:
		DPRINTF("%s: VSI %d: RSS not supported for VSI type %d\n",
		    __func__, vsi->idx, vsi->type);
		break;
	}
}

/**
 * ice_setup_vsi_qmap - Setup the queue mapping for a VSI
 * @vsi: the VSI to configure
 * @ctx: the VSI context to configure
 *
 * Configures the context for the given VSI, setting up how the firmware
 * should map the queues for this VSI.
 *
 * @pre vsi->qmap_type is set to a valid type
 */
int
ice_setup_vsi_qmap(struct ice_vsi *vsi, struct ice_vsi_ctx *ctx)
{
	int pow = 0;
	uint16_t qmap;

	KASSERT(vsi->rx_qmap != NULL);

	switch (vsi->qmap_type) {
	case ICE_RESMGR_ALLOC_CONTIGUOUS:
		ctx->info.mapping_flags |= htole16(ICE_AQ_VSI_Q_MAP_CONTIG);

		ctx->info.q_mapping[0] = htole16(vsi->rx_qmap[0]);
		ctx->info.q_mapping[1] = htole16(vsi->num_rx_queues);

		break;
	case ICE_RESMGR_ALLOC_SCATTERED:
		ctx->info.mapping_flags |= htole16(ICE_AQ_VSI_Q_MAP_NONCONTIG);

		for (int i = 0; i < vsi->num_rx_queues; i++)
			ctx->info.q_mapping[i] = htole16(vsi->rx_qmap[i]);
		break;
	default:
		return (EOPNOTSUPP);
	}

	/* Calculate the next power-of-2 of number of queues */
	if (vsi->num_rx_queues)
		pow = flsl(vsi->num_rx_queues - 1);

	/* Assign all the queues to traffic class zero */
	qmap = (pow << ICE_AQ_VSI_TC_Q_NUM_S) & ICE_AQ_VSI_TC_Q_NUM_M;
	ctx->info.tc_mapping[0] = htole16(qmap);

	/* Fill out default driver TC queue info for VSI */
	vsi->tc_info[0].qoffset = 0;
	vsi->tc_info[0].qcount_rx = vsi->num_rx_queues;
	vsi->tc_info[0].qcount_tx = vsi->num_tx_queues;
	for (int i = 1; i < ICE_MAX_TRAFFIC_CLASS; i++) {
		vsi->tc_info[i].qoffset = 0;
		vsi->tc_info[i].qcount_rx = 1;
		vsi->tc_info[i].qcount_tx = 1;
	}
	vsi->tc_map = 0x1;

	return 0;
}

/**
 * ice_aq_add_vsi
 * @hw: pointer to the HW struct
 * @vsi_ctx: pointer to a VSI context struct
 * @cd: pointer to command details structure or NULL
 *
 * Add a VSI context to the hardware (0x0210)
 */
enum ice_status
ice_aq_add_vsi(struct ice_hw *hw, struct ice_vsi_ctx *vsi_ctx,
	       struct ice_sq_cd *cd)
{
	struct ice_aqc_add_update_free_vsi_resp *res;
	struct ice_aqc_add_get_update_free_vsi *cmd;
	struct ice_aq_desc desc;
	enum ice_status status;

	cmd = &desc.params.vsi_cmd;
	res = &desc.params.add_update_free_vsi_res;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_add_vsi);

	if (!vsi_ctx->alloc_from_pool)
		cmd->vsi_num = htole16(vsi_ctx->vsi_num |
					   ICE_AQ_VSI_IS_VALID);
	cmd->vf_id = vsi_ctx->vf_num;

	cmd->vsi_flags = htole16(vsi_ctx->flags);

	desc.flags |= htole16(ICE_AQ_FLAG_RD);

	status = ice_aq_send_cmd(hw, &desc, &vsi_ctx->info,
				 sizeof(vsi_ctx->info), cd);

	if (!status) {
		vsi_ctx->vsi_num = le16toh(res->vsi_num) & ICE_AQ_VSI_NUM_M;
		vsi_ctx->vsis_allocd = le16toh(res->vsi_used);
		vsi_ctx->vsis_unallocated = le16toh(res->vsi_free);
	}

	return status;
}

/**
 * ice_add_vsi - add VSI context to the hardware and VSI handle list
 * @hw: pointer to the HW struct
 * @vsi_handle: unique VSI handle provided by drivers
 * @vsi_ctx: pointer to a VSI context struct
 * @cd: pointer to command details structure or NULL
 *
 * Add a VSI context to the hardware also add it into the VSI handle list.
 * If this function gets called after reset for existing VSIs then update
 * with the new HW VSI number in the corresponding VSI handle list entry.
 */
enum ice_status
ice_add_vsi(struct ice_hw *hw, uint16_t vsi_handle, struct ice_vsi_ctx *vsi_ctx,
	    struct ice_sq_cd *cd)
{
	struct ice_vsi_ctx *tmp_vsi_ctx;
	enum ice_status status;

	if (vsi_handle >= ICE_MAX_VSI)
		return ICE_ERR_PARAM;
	status = ice_aq_add_vsi(hw, vsi_ctx, cd);
	if (status)
		return status;
	tmp_vsi_ctx = ice_get_vsi_ctx(hw, vsi_handle);
	if (!tmp_vsi_ctx) {
		/* Create a new VSI context */
		tmp_vsi_ctx = (struct ice_vsi_ctx *)
			ice_malloc(hw, sizeof(*tmp_vsi_ctx));
		if (!tmp_vsi_ctx) {
			ice_aq_free_vsi(hw, vsi_ctx, false, cd);
			return ICE_ERR_NO_MEMORY;
		}
		*tmp_vsi_ctx = *vsi_ctx;

		hw->vsi_ctx[vsi_handle] = tmp_vsi_ctx;
	} else {
		/* update with new HW VSI num */
		tmp_vsi_ctx->vsi_num = vsi_ctx->vsi_num;
	}

	return ICE_SUCCESS;
}

/**
 * ice_aq_suspend_sched_elems - suspend scheduler elements
 * @hw: pointer to the HW struct
 * @elems_req: number of elements to suspend
 * @buf: pointer to buffer
 * @buf_size: buffer size in bytes
 * @elems_ret: returns total number of elements suspended
 * @cd: pointer to command details structure or NULL
 *
 * Suspend scheduling elements (0x0409)
 */
enum ice_status
ice_aq_suspend_sched_elems(struct ice_hw *hw, uint16_t elems_req, uint32_t *buf,
    uint16_t buf_size, uint16_t *elems_ret, struct ice_sq_cd *cd)
{
	return ice_aqc_send_sched_elem_cmd(hw, ice_aqc_opc_suspend_sched_elems,
					   elems_req, (void *)buf, buf_size,
					   elems_ret, cd);
}

/**
 * ice_aq_resume_sched_elems - resume scheduler elements
 * @hw: pointer to the HW struct
 * @elems_req: number of elements to resume
 * @buf: pointer to buffer
 * @buf_size: buffer size in bytes
 * @elems_ret: returns total number of elements resumed
 * @cd: pointer to command details structure or NULL
 *
 * resume scheduling elements (0x040A)
 */
enum ice_status
ice_aq_resume_sched_elems(struct ice_hw *hw, uint16_t elems_req, uint32_t *buf,
    uint16_t buf_size, uint16_t *elems_ret, struct ice_sq_cd *cd)
{
	return ice_aqc_send_sched_elem_cmd(hw, ice_aqc_opc_resume_sched_elems,
					   elems_req, (void *)buf, buf_size,
					   elems_ret, cd);
}

/**
 * ice_sched_suspend_resume_elems - suspend or resume HW nodes
 * @hw: pointer to the HW struct
 * @num_nodes: number of nodes
 * @node_teids: array of node teids to be suspended or resumed
 * @suspend: true means suspend / false means resume
 *
 * This function suspends or resumes HW nodes
 */
enum ice_status
ice_sched_suspend_resume_elems(struct ice_hw *hw, uint8_t num_nodes,
    uint32_t *node_teids, bool suspend)
{
	uint16_t i, buf_size, num_elem_ret = 0;
	enum ice_status status;
	uint32_t *buf;

	buf_size = sizeof(*buf) * num_nodes;
	buf = (uint32_t *)ice_malloc(hw, buf_size);
	if (!buf)
		return ICE_ERR_NO_MEMORY;

	for (i = 0; i < num_nodes; i++)
		buf[i] = htole32(node_teids[i]);

	if (suspend)
		status = ice_aq_suspend_sched_elems(hw, num_nodes, buf,
						    buf_size, &num_elem_ret,
						    NULL);
	else
		status = ice_aq_resume_sched_elems(hw, num_nodes, buf,
						   buf_size, &num_elem_ret,
						   NULL);
	if (status != ICE_SUCCESS || num_elem_ret != num_nodes)
		DNPRINTF(ICE_DBG_SCHED, "%s: suspend/resume failed\n",
		    __func__);

	ice_free(hw, buf);
	return status;
}

/**
 * ice_sched_calc_vsi_support_nodes - calculate number of VSI support nodes
 * @pi: pointer to the port info structure
 * @tc_node: pointer to TC node
 * @num_nodes: pointer to num nodes array
 *
 * This function calculates the number of supported nodes needed to add this
 * VSI into Tx tree including the VSI, parent and intermediate nodes in below
 * layers
 */
void
ice_sched_calc_vsi_support_nodes(struct ice_port_info *pi,
    struct ice_sched_node *tc_node, uint16_t *num_nodes)
{
	struct ice_sched_node *node;
	uint8_t vsil;
	int i;

	vsil = ice_sched_get_vsi_layer(pi->hw);
	for (i = vsil; i >= pi->hw->sw_entry_point_layer; i--) {
		/* Add intermediate nodes if TC has no children and
		 * need at least one node for VSI
		 */
		if (!tc_node->num_children || i == vsil) {
			num_nodes[i]++;
		} else {
			/* If intermediate nodes are reached max children
			 * then add a new one.
			 */
			node = ice_sched_get_first_node(pi, tc_node, (uint8_t)i);
			/* scan all the siblings */
			while (node) {
				if (node->num_children <
				    pi->hw->max_children[i])
					break;
				node = node->sibling;
			}

			/* tree has one intermediate node to add this new VSI.
			 * So no need to calculate supported nodes for below
			 * layers.
			 */
			if (node)
				break;
			/* all the nodes are full, allocate a new one */
			num_nodes[i]++;
		}
	}
}

/**
 * ice_sched_add_elems - add nodes to HW and SW DB
 * @pi: port information structure
 * @tc_node: pointer to the branch node
 * @parent: pointer to the parent node
 * @layer: layer number to add nodes
 * @num_nodes: number of nodes
 * @num_nodes_added: pointer to num nodes added
 * @first_node_teid: if new nodes are added then return the TEID of first node
 * @prealloc_nodes: preallocated nodes struct for software DB
 *
 * This function add nodes to HW as well as to SW DB for a given layer
 */
enum ice_status
ice_sched_add_elems(struct ice_port_info *pi, struct ice_sched_node *tc_node,
    struct ice_sched_node *parent, uint8_t layer, uint16_t num_nodes,
    uint16_t *num_nodes_added, uint32_t *first_node_teid,
    struct ice_sched_node **prealloc_nodes)
{
	struct ice_sched_node *prev, *new_node;
	struct ice_aqc_add_elem *buf;
	uint16_t i, num_groups_added = 0;
	enum ice_status status = ICE_SUCCESS;
	struct ice_hw *hw = pi->hw;
	uint16_t buf_size;
	uint32_t teid;

	buf_size = ice_struct_size(buf, generic, num_nodes);
	buf = (struct ice_aqc_add_elem *)ice_malloc(hw, buf_size);
	if (!buf)
		return ICE_ERR_NO_MEMORY;

	buf->hdr.parent_teid = parent->info.node_teid;
	buf->hdr.num_elems = htole16(num_nodes);
	for (i = 0; i < num_nodes; i++) {
		buf->generic[i].parent_teid = parent->info.node_teid;
		buf->generic[i].data.elem_type = ICE_AQC_ELEM_TYPE_SE_GENERIC;
		buf->generic[i].data.valid_sections =
			ICE_AQC_ELEM_VALID_GENERIC | ICE_AQC_ELEM_VALID_CIR |
			ICE_AQC_ELEM_VALID_EIR;
		buf->generic[i].data.generic = 0;
		buf->generic[i].data.cir_bw.bw_profile_idx =
			htole16(ICE_SCHED_DFLT_RL_PROF_ID);
		buf->generic[i].data.cir_bw.bw_alloc =
			htole16(ICE_SCHED_DFLT_BW_WT);
		buf->generic[i].data.eir_bw.bw_profile_idx =
			htole16(ICE_SCHED_DFLT_RL_PROF_ID);
		buf->generic[i].data.eir_bw.bw_alloc =
			htole16(ICE_SCHED_DFLT_BW_WT);
	}

	status = ice_aq_add_sched_elems(hw, 1, buf, buf_size,
					&num_groups_added, NULL);
	if (status != ICE_SUCCESS || num_groups_added != 1) {
		DNPRINTF(ICE_DBG_SCHED, "%s: add node failed FW Error %d\n",
			  __func__, hw->adminq.sq_last_status);
		ice_free(hw, buf);
		return ICE_ERR_CFG;
	}

	*num_nodes_added = num_nodes;
	/* add nodes to the SW DB */
	for (i = 0; i < num_nodes; i++) {
		if (prealloc_nodes) {
			status = ice_sched_add_node(pi, layer,
			    &buf->generic[i], prealloc_nodes[i]);
		} else {
			status = ice_sched_add_node(pi, layer,
			    &buf->generic[i], NULL);
		}

		if (status != ICE_SUCCESS) {
			DNPRINTF(ICE_DBG_SCHED,
			    "%s: add nodes in SW DB failed status =%d\n",
			    __func__, status);
			break;
		}

		teid = le32toh(buf->generic[i].node_teid);
		new_node = ice_sched_find_node_by_teid(parent, teid);
		if (!new_node) {
			DNPRINTF(ICE_DBG_SCHED,
			    "%s: Node is missing for teid =%d\n",
			    __func__, teid);
			break;
		}

		new_node->sibling = NULL;
		new_node->tc_num = tc_node->tc_num;

		/* add it to previous node sibling pointer */
		/* Note: siblings are not linked across branches */
		prev = ice_sched_get_first_node(pi, tc_node, layer);
		if (prev && prev != new_node) {
			while (prev->sibling)
				prev = prev->sibling;
			prev->sibling = new_node;
		}

		/* initialize the sibling head */
		if (!pi->sib_head[tc_node->tc_num][layer])
			pi->sib_head[tc_node->tc_num][layer] = new_node;

		if (i == 0)
			*first_node_teid = teid;
	}

	ice_free(hw, buf);
	return status;
}

/**
 * ice_sched_add_nodes_to_hw_layer - Add nodes to hw layer
 * @pi: port information structure
 * @tc_node: pointer to TC node
 * @parent: pointer to parent node
 * @layer: layer number to add nodes
 * @num_nodes: number of nodes to be added
 * @first_node_teid: pointer to the first node TEID
 * @num_nodes_added: pointer to number of nodes added
 *
 * Add nodes into specific hw layer.
 */
enum ice_status
ice_sched_add_nodes_to_hw_layer(struct ice_port_info *pi,
				struct ice_sched_node *tc_node,
				struct ice_sched_node *parent, uint8_t layer,
				uint16_t num_nodes, uint32_t *first_node_teid,
				uint16_t *num_nodes_added)
{
	uint16_t max_child_nodes;

	*num_nodes_added = 0;

	if (!num_nodes)
		return ICE_SUCCESS;

	if (!parent || layer < pi->hw->sw_entry_point_layer)
		return ICE_ERR_PARAM;

	/* max children per node per layer */
	max_child_nodes = pi->hw->max_children[parent->tx_sched_layer];

	/* current number of children + required nodes exceed max children */
	if ((parent->num_children + num_nodes) > max_child_nodes) {
		/* Fail if the parent is a TC node */
		if (parent == tc_node)
			return ICE_ERR_CFG;
		return ICE_ERR_MAX_LIMIT;
	}

	return ice_sched_add_elems(pi, tc_node, parent, layer, num_nodes,
				   num_nodes_added, first_node_teid, NULL);
}

/**
 * ice_sched_add_nodes_to_layer - Add nodes to a given layer
 * @pi: port information structure
 * @tc_node: pointer to TC node
 * @parent: pointer to parent node
 * @layer: layer number to add nodes
 * @num_nodes: number of nodes to be added
 * @first_node_teid: pointer to the first node TEID
 * @num_nodes_added: pointer to number of nodes added
 *
 * This function add nodes to a given layer.
 */
enum ice_status
ice_sched_add_nodes_to_layer(struct ice_port_info *pi,
			     struct ice_sched_node *tc_node,
			     struct ice_sched_node *parent, uint8_t layer,
			     uint16_t num_nodes, uint32_t *first_node_teid,
			     uint16_t *num_nodes_added)
{
	uint32_t *first_teid_ptr = first_node_teid;
	uint16_t new_num_nodes = num_nodes;
	enum ice_status status = ICE_SUCCESS;
	uint32_t temp;

	*num_nodes_added = 0;
	while (*num_nodes_added < num_nodes) {
		uint16_t max_child_nodes, num_added = 0;

		status = ice_sched_add_nodes_to_hw_layer(pi, tc_node, parent,
							 layer,	new_num_nodes,
							 first_teid_ptr,
							 &num_added);
		if (status == ICE_SUCCESS)
			*num_nodes_added += num_added;
		/* added more nodes than requested ? */
		if (*num_nodes_added > num_nodes) {
			DNPRINTF(ICE_DBG_SCHED, "%s: added extra nodes %d %d\n",
			    __func__, num_nodes, *num_nodes_added);
			status = ICE_ERR_CFG;
			break;
		}
		/* break if all the nodes are added successfully */
		if (status == ICE_SUCCESS && (*num_nodes_added == num_nodes))
			break;
		/* break if the error is not max limit */
		if (status != ICE_SUCCESS && status != ICE_ERR_MAX_LIMIT)
			break;
		/* Exceeded the max children */
		max_child_nodes = pi->hw->max_children[parent->tx_sched_layer];
		/* utilize all the spaces if the parent is not full */
		if (parent->num_children < max_child_nodes) {
			new_num_nodes = max_child_nodes - parent->num_children;
		} else {
			/* This parent is full, try the next sibling */
			parent = parent->sibling;
			/* Don't modify the first node TEID memory if the
			 * first node was added already in the above call.
			 * Instead send some temp memory for all other
			 * recursive calls.
			 */
			if (num_added)
				first_teid_ptr = &temp;

			new_num_nodes = num_nodes - *num_nodes_added;
		}
	}

	return status;
}

/**
 * ice_sched_add_vsi_support_nodes - add VSI supported nodes into Tx tree
 * @pi: port information structure
 * @vsi_handle: software VSI handle
 * @tc_node: pointer to TC node
 * @num_nodes: pointer to num nodes array
 *
 * This function adds the VSI supported nodes into Tx tree including the
 * VSI, its parent and intermediate nodes in below layers
 */
enum ice_status
ice_sched_add_vsi_support_nodes(struct ice_port_info *pi, uint16_t vsi_handle,
    struct ice_sched_node *tc_node, uint16_t *num_nodes)
{
	struct ice_sched_node *parent = tc_node;
	uint32_t first_node_teid;
	uint16_t num_added = 0;
	uint8_t i, vsil;

	if (!pi)
		return ICE_ERR_PARAM;

	vsil = ice_sched_get_vsi_layer(pi->hw);
	for (i = pi->hw->sw_entry_point_layer; i <= vsil; i++) {
		enum ice_status status;

		status = ice_sched_add_nodes_to_layer(pi, tc_node, parent,
						      i, num_nodes[i],
						      &first_node_teid,
						      &num_added);
		if (status != ICE_SUCCESS || num_nodes[i] != num_added)
			return ICE_ERR_CFG;

		/* The newly added node can be a new parent for the next
		 * layer nodes
		 */
		if (num_added)
			parent = ice_sched_find_node_by_teid(tc_node,
							     first_node_teid);
		else
			parent = parent->children[0];

		if (!parent)
			return ICE_ERR_CFG;

		if (i == vsil)
			parent->vsi_handle = vsi_handle;
	}

	return ICE_SUCCESS;
}

/**
 * ice_sched_add_vsi_to_topo - add a new VSI into tree
 * @pi: port information structure
 * @vsi_handle: software VSI handle
 * @tc: TC number
 *
 * This function adds a new VSI into scheduler tree
 */
enum ice_status
ice_sched_add_vsi_to_topo(struct ice_port_info *pi, uint16_t vsi_handle,
    uint8_t tc)
{
	uint16_t num_nodes[ICE_AQC_TOPO_MAX_LEVEL_NUM] = { 0 };
	struct ice_sched_node *tc_node;

	tc_node = ice_sched_get_tc_node(pi, tc);
	if (!tc_node)
		return ICE_ERR_PARAM;

	/* calculate number of supported nodes needed for this VSI */
	ice_sched_calc_vsi_support_nodes(pi, tc_node, num_nodes);

	/* add VSI supported nodes to TC subtree */
	return ice_sched_add_vsi_support_nodes(pi, vsi_handle, tc_node,
					       num_nodes);
}

/**
 * ice_alloc_lan_q_ctx - allocate LAN queue contexts for the given VSI and TC
 * @hw: pointer to the HW struct
 * @vsi_handle: VSI handle
 * @tc: TC number
 * @new_numqs: number of queues
 */
enum ice_status
ice_alloc_lan_q_ctx(struct ice_hw *hw, uint16_t vsi_handle, uint8_t tc,
    uint16_t new_numqs)
{
	struct ice_vsi_ctx *vsi_ctx;
	struct ice_q_ctx *q_ctx;

	vsi_ctx = ice_get_vsi_ctx(hw, vsi_handle);
	if (!vsi_ctx)
		return ICE_ERR_PARAM;

	/* allocate LAN queue contexts */
	if (!vsi_ctx->lan_q_ctx[tc]) {
		vsi_ctx->lan_q_ctx[tc] = (struct ice_q_ctx *)
			ice_calloc(hw, new_numqs, sizeof(*q_ctx));
		if (!vsi_ctx->lan_q_ctx[tc])
			return ICE_ERR_NO_MEMORY;
		vsi_ctx->num_lan_q_entries[tc] = new_numqs;
		return ICE_SUCCESS;
	}

	/* num queues are increased, update the queue contexts */
	if (new_numqs > vsi_ctx->num_lan_q_entries[tc]) {
		uint16_t prev_num = vsi_ctx->num_lan_q_entries[tc];

		q_ctx = (struct ice_q_ctx *)
			ice_calloc(hw, new_numqs, sizeof(*q_ctx));
		if (!q_ctx)
			return ICE_ERR_NO_MEMORY;
		memcpy(q_ctx, vsi_ctx->lan_q_ctx[tc],
		    prev_num * sizeof(*q_ctx));
		ice_free(hw, vsi_ctx->lan_q_ctx[tc]);
		vsi_ctx->lan_q_ctx[tc] = q_ctx;
		vsi_ctx->num_lan_q_entries[tc] = new_numqs;
	}

	return ICE_SUCCESS;
}

/**
 * ice_sched_calc_vsi_child_nodes - calculate number of VSI child nodes
 * @hw: pointer to the HW struct
 * @num_qs: number of queues
 * @num_nodes: num nodes array
 *
 * This function calculates the number of VSI child nodes based on the
 * number of queues.
 */
void
ice_sched_calc_vsi_child_nodes(struct ice_hw *hw, uint16_t num_qs, uint16_t *num_nodes)
{
	uint16_t num = num_qs;
	uint8_t i, qgl, vsil;

	qgl = ice_sched_get_qgrp_layer(hw);
	vsil = ice_sched_get_vsi_layer(hw);

	/* calculate num nodes from queue group to VSI layer */
	for (i = qgl; i > vsil; i--) {
		/* round to the next integer if there is a remainder */
		num = howmany(num, hw->max_children[i]);

		/* need at least one node */
		num_nodes[i] = num ? num : 1;
	}
}

/**
 * ice_sched_add_vsi_child_nodes - add VSI child nodes to tree
 * @pi: port information structure
 * @vsi_handle: software VSI handle
 * @tc_node: pointer to the TC node
 * @num_nodes: pointer to the num nodes that needs to be added per layer
 * @owner: node owner (LAN or RDMA)
 *
 * This function adds the VSI child nodes to tree. It gets called for
 * LAN and RDMA separately.
 */
enum ice_status
ice_sched_add_vsi_child_nodes(struct ice_port_info *pi, uint16_t vsi_handle,
    struct ice_sched_node *tc_node, uint16_t *num_nodes, uint8_t owner)
{
	struct ice_sched_node *parent, *node;
	struct ice_hw *hw = pi->hw;
	uint32_t first_node_teid;
	uint16_t num_added = 0;
	uint8_t i, qgl, vsil;

	qgl = ice_sched_get_qgrp_layer(hw);
	vsil = ice_sched_get_vsi_layer(hw);
	parent = ice_sched_get_vsi_node(pi, tc_node, vsi_handle);
	for (i = vsil + 1; i <= qgl; i++) {
		enum ice_status status;

		if (!parent)
			return ICE_ERR_CFG;

		status = ice_sched_add_nodes_to_layer(pi, tc_node, parent, i,
						      num_nodes[i],
						      &first_node_teid,
						      &num_added);
		if (status != ICE_SUCCESS || num_nodes[i] != num_added)
			return ICE_ERR_CFG;

		/* The newly added node can be a new parent for the next
		 * layer nodes
		 */
		if (num_added) {
			parent = ice_sched_find_node_by_teid(tc_node,
							     first_node_teid);
			node = parent;
			while (node) {
				node->owner = owner;
				node = node->sibling;
			}
		} else {
			parent = parent->children[0];
		}
	}

	return ICE_SUCCESS;
}

/**
 * ice_sched_update_vsi_child_nodes - update VSI child nodes
 * @pi: port information structure
 * @vsi_handle: software VSI handle
 * @tc: TC number
 * @new_numqs: new number of max queues
 * @owner: owner of this subtree
 *
 * This function updates the VSI child nodes based on the number of queues
 */
enum ice_status
ice_sched_update_vsi_child_nodes(struct ice_port_info *pi, uint16_t vsi_handle,
				 uint8_t tc, uint16_t new_numqs, uint8_t owner)
{
	uint16_t new_num_nodes[ICE_AQC_TOPO_MAX_LEVEL_NUM] = { 0 };
	struct ice_sched_node *vsi_node;
	struct ice_sched_node *tc_node;
	struct ice_vsi_ctx *vsi_ctx;
	enum ice_status status = ICE_SUCCESS;
	struct ice_hw *hw = pi->hw;
	uint16_t prev_numqs;

	tc_node = ice_sched_get_tc_node(pi, tc);
	if (!tc_node)
		return ICE_ERR_CFG;

	vsi_node = ice_sched_get_vsi_node(pi, tc_node, vsi_handle);
	if (!vsi_node)
		return ICE_ERR_CFG;

	vsi_ctx = ice_get_vsi_ctx(hw, vsi_handle);
	if (!vsi_ctx)
		return ICE_ERR_PARAM;

	if (owner == ICE_SCHED_NODE_OWNER_LAN)
		prev_numqs = vsi_ctx->sched.max_lanq[tc];
	else
		prev_numqs = vsi_ctx->sched.max_rdmaq[tc];
	/* num queues are not changed or less than the previous number */
	if (new_numqs <= prev_numqs)
		return status;
	if (owner == ICE_SCHED_NODE_OWNER_LAN) {
		status = ice_alloc_lan_q_ctx(hw, vsi_handle, tc, new_numqs);
		if (status)
			return status;
	} else {
#if 0
		status = ice_alloc_rdma_q_ctx(hw, vsi_handle, tc, new_numqs);
		if (status)
			return status;
#else
		return ICE_ERR_NOT_IMPL;
#endif
	}

	if (new_numqs)
		ice_sched_calc_vsi_child_nodes(hw, new_numqs, new_num_nodes);
	/* Keep the max number of queue configuration all the time. Update the
	 * tree only if number of queues > previous number of queues. This may
	 * leave some extra nodes in the tree if number of queues < previous
	 * number but that wouldn't harm anything. Removing those extra nodes
	 * may complicate the code if those nodes are part of SRL or
	 * individually rate limited.
	 */
	status = ice_sched_add_vsi_child_nodes(pi, vsi_handle, tc_node,
					       new_num_nodes, owner);
	if (status)
		return status;
	if (owner == ICE_SCHED_NODE_OWNER_LAN)
		vsi_ctx->sched.max_lanq[tc] = new_numqs;
	else
		vsi_ctx->sched.max_rdmaq[tc] = new_numqs;

	return ICE_SUCCESS;
}

/**
 * ice_sched_cfg_vsi - configure the new/existing VSI
 * @pi: port information structure
 * @vsi_handle: software VSI handle
 * @tc: TC number
 * @maxqs: max number of queues
 * @owner: LAN or RDMA
 * @enable: TC enabled or disabled
 *
 * This function adds/updates VSI nodes based on the number of queues. If TC is
 * enabled and VSI is in suspended state then resume the VSI back. If TC is
 * disabled then suspend the VSI if it is not already.
 */
enum ice_status
ice_sched_cfg_vsi(struct ice_port_info *pi, uint16_t vsi_handle, uint8_t tc,
    uint16_t maxqs, uint8_t owner, bool enable)
{
	struct ice_sched_node *vsi_node, *tc_node;
	struct ice_vsi_ctx *vsi_ctx;
	enum ice_status status = ICE_SUCCESS;
	struct ice_hw *hw = pi->hw;

	DNPRINTF(ICE_DBG_SCHED, "%s: add/config VSI %d\n",
	    __func__, vsi_handle);
	tc_node = ice_sched_get_tc_node(pi, tc);
	if (!tc_node)
		return ICE_ERR_PARAM;
	vsi_ctx = ice_get_vsi_ctx(hw, vsi_handle);
	if (!vsi_ctx)
		return ICE_ERR_PARAM;
	vsi_node = ice_sched_get_vsi_node(pi, tc_node, vsi_handle);

	/* suspend the VSI if TC is not enabled */
	if (!enable) {
		if (vsi_node && vsi_node->in_use) {
			uint32_t teid = le32toh(vsi_node->info.node_teid);

			status = ice_sched_suspend_resume_elems(hw, 1, &teid,
								true);
			if (!status)
				vsi_node->in_use = false;
		}
		return status;
	}

	/* TC is enabled, if it is a new VSI then add it to the tree */
	if (!vsi_node) {
		status = ice_sched_add_vsi_to_topo(pi, vsi_handle, tc);
		if (status)
			return status;

		vsi_node = ice_sched_get_vsi_node(pi, tc_node, vsi_handle);
		if (!vsi_node)
			return ICE_ERR_CFG;

		vsi_ctx->sched.vsi_node[tc] = vsi_node;
		vsi_node->in_use = true;
		/* invalidate the max queues whenever VSI gets added first time
		 * into the scheduler tree (boot or after reset). We need to
		 * recreate the child nodes all the time in these cases.
		 */
		vsi_ctx->sched.max_lanq[tc] = 0;
		vsi_ctx->sched.max_rdmaq[tc] = 0;
	}

	/* update the VSI child nodes */
	status = ice_sched_update_vsi_child_nodes(pi, vsi_handle, tc, maxqs,
						  owner);
	if (status)
		return status;

	/* TC is enabled, resume the VSI if it is in the suspend state */
	if (!vsi_node->in_use) {
		uint32_t teid = le32toh(vsi_node->info.node_teid);

		status = ice_sched_suspend_resume_elems(hw, 1, &teid, false);
		if (!status)
			vsi_node->in_use = true;
	}

	return status;
}

static inline bool ice_is_tc_ena(ice_bitmap_t bitmap, uint8_t tc)
{
	return !!(bitmap & BIT(tc));
}

/**
 * ice_cfg_vsi_qs - configure the new/existing VSI queues
 * @pi: port information structure
 * @vsi_handle: software VSI handle
 * @tc_bitmap: TC bitmap
 * @maxqs: max queues array per TC
 * @owner: LAN or RDMA
 *
 * This function adds/updates the VSI queues per TC.
 */
enum ice_status
ice_cfg_vsi_qs(struct ice_port_info *pi, uint16_t vsi_handle,
    uint16_t tc_bitmap, uint16_t *maxqs, uint8_t owner)
{
	enum ice_status status = ICE_SUCCESS;
	uint8_t i;

	if (!pi || pi->port_state != ICE_SCHED_PORT_STATE_READY)
		return ICE_ERR_CFG;

	if (!ice_is_vsi_valid(pi->hw, vsi_handle))
		return ICE_ERR_PARAM;
#if 0
	ice_acquire_lock(&pi->sched_lock);
#endif
	ice_for_each_traffic_class(i) {
		/* configuration is possible only if TC node is present */
		if (!ice_sched_get_tc_node(pi, i))
			continue;

		status = ice_sched_cfg_vsi(pi, vsi_handle, i, maxqs[i], owner,
					   ice_is_tc_ena(tc_bitmap, i));
		if (status)
			break;
	}
#if 0
	ice_release_lock(&pi->sched_lock);
#endif
	return status;
}

/**
 * ice_cfg_vsi_lan - configure VSI LAN queues
 * @pi: port information structure
 * @vsi_handle: software VSI handle
 * @tc_bitmap: TC bitmap
 * @max_lanqs: max LAN queues array per TC
 *
 * This function adds/updates the VSI LAN queues per TC.
 */
enum ice_status
ice_cfg_vsi_lan(struct ice_port_info *pi, uint16_t vsi_handle,
    uint16_t tc_bitmap, uint16_t *max_lanqs)
{
	return ice_cfg_vsi_qs(pi, vsi_handle, tc_bitmap, max_lanqs,
			      ICE_SCHED_NODE_OWNER_LAN);
}

/**
 * ice_reset_vsi_stats - Reset VSI statistics counters
 * @vsi: VSI structure
 *
 * Resets the software tracking counters for the VSI statistics, and indicate
 * that the offsets haven't been loaded. This is intended to be called
 * post-reset so that VSI statistics count from zero again.
 */
void
ice_reset_vsi_stats(struct ice_vsi *vsi)
{
	/* Reset HW stats */
	memset(&vsi->hw_stats.prev, 0, sizeof(vsi->hw_stats.prev));
	memset(&vsi->hw_stats.cur, 0, sizeof(vsi->hw_stats.cur));
	vsi->hw_stats.offsets_loaded = false;
}

/**
 * ice_initialize_vsi - Initialize a VSI for use
 * @vsi: the vsi to initialize
 *
 * Initialize a VSI over the adminq and prepare it for operation.
 *
 * @pre vsi->num_tx_queues is set
 * @pre vsi->num_rx_queues is set
 */
int
ice_initialize_vsi(struct ice_vsi *vsi)
{
	struct ice_vsi_ctx ctx = { 0 };
	struct ice_hw *hw = &vsi->sc->hw;
	uint16_t max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 };
	enum ice_status status;
	int err;

	/* For now, we only have code supporting PF VSIs */
	switch (vsi->type) {
	case ICE_VSI_PF:
		ctx.flags = ICE_AQ_VSI_TYPE_PF;
		break;
	case ICE_VSI_VMDQ2:
		ctx.flags = ICE_AQ_VSI_TYPE_VMDQ2;
		break;
	default:
		return (ENODEV);
	}

	ice_set_default_vsi_ctx(&ctx);
	ice_set_rss_vsi_ctx(&ctx, vsi->type);

	/* XXX: VSIs of other types may need different port info? */
	ctx.info.sw_id = hw->port_info->sw_id;

	/* Set some RSS parameters based on the VSI type */
	ice_vsi_set_rss_params(vsi);

	/* Initialize the Rx queue mapping for this VSI */
	err = ice_setup_vsi_qmap(vsi, &ctx);
	if (err)
		return err;

	/* (Re-)add VSI to HW VSI handle list */
	status = ice_add_vsi(hw, vsi->idx, &ctx, NULL);
	if (status != 0) {
		DPRINTF("%s: Add VSI AQ call failed, err %s aq_err %s\n",
		    __func__, ice_status_str(status),
		    ice_aq_str(hw->adminq.sq_last_status));
		return (EIO);
	}
	vsi->info = ctx.info;

	/* Initialize VSI with just 1 TC to start */
	max_txqs[0] = vsi->num_tx_queues;

	status = ice_cfg_vsi_lan(hw->port_info, vsi->idx,
	    ICE_DFLT_TRAFFIC_CLASS, max_txqs);
	if (status) {
		DPRINTF("%s: Failed VSI lan queue config, err %s aq_err %s\n",
		    __func__, ice_status_str(status),
		    ice_aq_str(hw->adminq.sq_last_status));
		ice_deinit_vsi(vsi);
		return (ENODEV);
	}

	/* Reset VSI stats */
	ice_reset_vsi_stats(vsi);

	return 0;
}

/**
 * ice_sched_replay_agg - recreate aggregator node(s)
 * @hw: pointer to the HW struct
 *
 * This function recreate aggregator type nodes which are not replayed earlier.
 * It also replay aggregator BW information. These aggregator nodes are not
 * associated with VSI type node yet.
 */
void
ice_sched_replay_agg(struct ice_hw *hw)
{
#if 0
	struct ice_port_info *pi = hw->port_info;
	struct ice_sched_agg_info *agg_info;

	ice_acquire_lock(&pi->sched_lock);
	LIST_FOR_EACH_ENTRY(agg_info, &hw->agg_list, ice_sched_agg_info,
			    list_entry)
		/* replay aggregator (re-create aggregator node) */
		if (!ice_cmp_bitmap(agg_info->tc_bitmap,
				    agg_info->replay_tc_bitmap,
				    ICE_MAX_TRAFFIC_CLASS)) {
			ice_declare_bitmap(replay_bitmap,
					   ICE_MAX_TRAFFIC_CLASS);
			enum ice_status status;

			ice_zero_bitmap(replay_bitmap, ICE_MAX_TRAFFIC_CLASS);
			ice_sched_get_ena_tc_bitmap(pi,
						    agg_info->replay_tc_bitmap,
						    replay_bitmap);
			status = ice_sched_cfg_agg(hw->port_info,
						   agg_info->agg_id,
						   ICE_AGG_TYPE_AGG,
						   replay_bitmap);
			if (status) {
				ice_info(hw, "Replay agg id[%d] failed\n",
					 agg_info->agg_id);
				/* Move on to next one */
				continue;
			}
			/* Replay aggregator node BW (restore aggregator BW) */
			status = ice_sched_replay_agg_bw(hw, agg_info);
			if (status)
				ice_info(hw, "Replay agg bw [id=%d] failed\n",
					 agg_info->agg_id);
		}
	ice_release_lock(&pi->sched_lock);
#endif
}

/**
 * ice_replay_post - post replay configuration cleanup
 * @hw: pointer to the HW struct
 *
 * Post replay cleanup.
 */
void ice_replay_post(struct ice_hw *hw)
{
	/* Delete old entries from replay filter list head */
	ice_rm_sw_replay_rule_info(hw, hw->switch_info);
	ice_sched_replay_agg(hw);
}

/**
 * ice_is_main_vsi - checks whether the VSI is main VSI
 * @hw: pointer to the HW struct
 * @vsi_handle: VSI handle
 *
 * Checks whether the VSI is the main VSI (the first PF VSI created on
 * given PF).
 */
bool
ice_is_main_vsi(struct ice_hw *hw, uint16_t vsi_handle)
{
	return vsi_handle == ICE_MAIN_VSI_HANDLE && hw->vsi_ctx[vsi_handle];
}

/**
 * ice_replay_pre_init - replay pre initialization
 * @hw: pointer to the HW struct
 * @sw: pointer to switch info struct for which function initializes filters
 *
 * Initializes required config data for VSI, FD, ACL, and RSS before replay.
 */
enum ice_status
ice_replay_pre_init(struct ice_hw *hw, struct ice_switch_info *sw)
{
#if 0
	enum ice_status status;
	uint8_t i;

	/* Delete old entries from replay filter list head if there is any */
	ice_rm_sw_replay_rule_info(hw, sw);
	/* In start of replay, move entries into replay_rules list, it
	 * will allow adding rules entries back to filt_rules list,
	 * which is operational list.
	 */
	for (i = 0; i < ICE_MAX_NUM_RECIPES; i++)
		LIST_REPLACE_INIT(&sw->recp_list[i].filt_rules,
				  &sw->recp_list[i].filt_replay_rules);
	ice_sched_replay_agg_vsi_preinit(hw);

	status = ice_sched_replay_root_node_bw(hw->port_info);
	if (status)
		return status;

	return ice_sched_replay_tc_node_bw(hw->port_info);
#else
	return ICE_ERR_NOT_IMPL;
#endif
}

/**
 * ice_replay_rss_cfg - replay RSS configurations associated with VSI
 * @hw: pointer to the hardware structure
 * @vsi_handle: software VSI handle
 */
enum ice_status
ice_replay_rss_cfg(struct ice_hw *hw, uint16_t vsi_handle)
{
#if 0
	enum ice_status status = ICE_SUCCESS;
	struct ice_rss_cfg *r;

	if (!ice_is_vsi_valid(hw, vsi_handle))
		return ICE_ERR_PARAM;

	ice_acquire_lock(&hw->rss_locks);
	LIST_FOR_EACH_ENTRY(r, &hw->rss_list_head,
			    ice_rss_cfg, l_entry) {
		if (ice_is_bit_set(r->vsis, vsi_handle)) {
			status = ice_add_rss_cfg_sync(hw, vsi_handle, &r->hash);
			if (status)
				break;
		}
	}
	ice_release_lock(&hw->rss_locks);

	return status;
#else
	return ICE_ERR_NOT_IMPL;
#endif
}

/**
 * ice_find_vsi_list_entry - Search VSI list map with VSI count 1
 * @recp_list: VSI lists needs to be searched
 * @vsi_handle: VSI handle to be found in VSI list
 * @vsi_list_id: VSI list ID found containing vsi_handle
 *
 * Helper function to search a VSI list with single entry containing given VSI
 * handle element. This can be extended further to search VSI list with more
 * than 1 vsi_count. Returns pointer to VSI list entry if found.
 */
struct ice_vsi_list_map_info *
ice_find_vsi_list_entry(struct ice_sw_recipe *recp_list, uint16_t vsi_handle,
			uint16_t *vsi_list_id)
{
	struct ice_vsi_list_map_info *map_info = NULL;

	if (recp_list->adv_rule) {
		struct ice_adv_fltr_mgmt_list_head *adv_list_head;
		struct ice_adv_fltr_mgmt_list_entry *list_itr;

		adv_list_head = &recp_list->adv_filt_rules;
		TAILQ_FOREACH(list_itr, adv_list_head, list_entry) {
			if (list_itr->vsi_list_info) {
				map_info = list_itr->vsi_list_info;
				if (ice_is_bit_set(map_info->vsi_map,
						   vsi_handle)) {
					*vsi_list_id = map_info->vsi_list_id;
					return map_info;
				}
			}
		}
	} else {
		struct ice_fltr_mgmt_list_head *list_head;
		struct ice_fltr_mgmt_list_entry *list_itr;

		list_head = &recp_list->filt_rules;
		TAILQ_FOREACH(list_itr, list_head, list_entry) {
			if (list_itr->vsi_count == 1 &&
			    list_itr->vsi_list_info) {
				map_info = list_itr->vsi_list_info;
				if (ice_is_bit_set(map_info->vsi_map,
						   vsi_handle)) {
					*vsi_list_id = map_info->vsi_list_id;
					return map_info;
				}
			}
		}
	}

	return NULL;
}

/**
 * ice_add_vlan_internal - Add one VLAN based filter rule
 * @hw: pointer to the hardware structure
 * @recp_list: recipe list for which rule has to be added
 * @f_entry: filter entry containing one VLAN information
 */
enum ice_status
ice_add_vlan_internal(struct ice_hw *hw, struct ice_sw_recipe *recp_list,
		      struct ice_fltr_list_entry *f_entry)
{
	struct ice_fltr_mgmt_list_entry *v_list_itr;
	struct ice_fltr_info *new_fltr, *cur_fltr;
	enum ice_sw_lkup_type lkup_type;
	uint16_t vsi_list_id = 0, vsi_handle;
#if 0
	struct ice_lock *rule_lock; /* Lock to protect filter rule list */
#endif
	enum ice_status status = ICE_SUCCESS;

	if (!ice_is_vsi_valid(hw, f_entry->fltr_info.vsi_handle))
		return ICE_ERR_PARAM;

	f_entry->fltr_info.fwd_id.hw_vsi_id =
		hw->vsi_ctx[f_entry->fltr_info.vsi_handle]->vsi_num;
	new_fltr = &f_entry->fltr_info;

	/* VLAN ID should only be 12 bits */
	if (new_fltr->l_data.vlan.vlan_id > ICE_MAX_VLAN_ID)
		return ICE_ERR_PARAM;

	if (new_fltr->src_id != ICE_SRC_ID_VSI)
		return ICE_ERR_PARAM;

	new_fltr->src = new_fltr->fwd_id.hw_vsi_id;
	lkup_type = new_fltr->lkup_type;
	vsi_handle = new_fltr->vsi_handle;
#if 0
	rule_lock = &recp_list->filt_rule_lock;
	ice_acquire_lock(rule_lock);
#endif
	v_list_itr = ice_find_rule_entry(&recp_list->filt_rules, new_fltr);
	if (!v_list_itr) {
		struct ice_vsi_list_map_info *map_info = NULL;

		if (new_fltr->fltr_act == ICE_FWD_TO_VSI) {
			/* All VLAN pruning rules use a VSI list. Check if
			 * there is already a VSI list containing VSI that we
			 * want to add. If found, use the same vsi_list_id for
			 * this new VLAN rule or else create a new list.
			 */
			map_info = ice_find_vsi_list_entry(recp_list,
							   vsi_handle,
							   &vsi_list_id);
			if (!map_info) {
				status = ice_create_vsi_list_rule(hw,
								  &vsi_handle,
								  1,
								  &vsi_list_id,
								  lkup_type);
				if (status)
					goto exit;
			}
			/* Convert the action to forwarding to a VSI list. */
			new_fltr->fltr_act = ICE_FWD_TO_VSI_LIST;
			new_fltr->fwd_id.vsi_list_id = vsi_list_id;
		}

		status = ice_create_pkt_fwd_rule(hw, recp_list, f_entry);
		if (!status) {
			v_list_itr = ice_find_rule_entry(&recp_list->filt_rules,
							 new_fltr);
			if (!v_list_itr) {
				status = ICE_ERR_DOES_NOT_EXIST;
				goto exit;
			}
			/* reuse VSI list for new rule and increment ref_cnt */
			if (map_info) {
				v_list_itr->vsi_list_info = map_info;
				map_info->ref_cnt++;
			} else {
				v_list_itr->vsi_list_info =
					ice_create_vsi_list_map(hw, &vsi_handle,
								1, vsi_list_id);
			}
		}
	} else if (v_list_itr->vsi_list_info->ref_cnt == 1) {
		/* Update existing VSI list to add new VSI ID only if it used
		 * by one VLAN rule.
		 */
		cur_fltr = &v_list_itr->fltr_info;
		status = ice_add_update_vsi_list(hw, v_list_itr, cur_fltr,
						 new_fltr);
	} else {
		/* If VLAN rule exists and VSI list being used by this rule is
		 * referenced by more than 1 VLAN rule. Then create a new VSI
		 * list appending previous VSI with new VSI and update existing
		 * VLAN rule to point to new VSI list ID
		 */
		struct ice_fltr_info tmp_fltr;
		uint16_t vsi_handle_arr[2];
		uint16_t cur_handle;

		/* Current implementation only supports reusing VSI list with
		 * one VSI count. We should never hit below condition
		 */
		if (v_list_itr->vsi_count > 1 &&
		    v_list_itr->vsi_list_info->ref_cnt > 1) {
			DNPRINTF(ICE_DBG_SW, "%s: Invalid configuration: "
			    "Optimization to reuse VSI list with more than "
			    "one VSI is not being done yet\n", __func__);
			status = ICE_ERR_CFG;
			goto exit;
		}

		cur_handle =
			ice_find_first_bit(v_list_itr->vsi_list_info->vsi_map,
					   ICE_MAX_VSI);

		/* A rule already exists with the new VSI being added */
		if (cur_handle == vsi_handle) {
			status = ICE_ERR_ALREADY_EXISTS;
			goto exit;
		}

		vsi_handle_arr[0] = cur_handle;
		vsi_handle_arr[1] = vsi_handle;
		status = ice_create_vsi_list_rule(hw, &vsi_handle_arr[0], 2,
						  &vsi_list_id, lkup_type);
		if (status)
			goto exit;

		tmp_fltr = v_list_itr->fltr_info;
		tmp_fltr.fltr_rule_id = v_list_itr->fltr_info.fltr_rule_id;
		tmp_fltr.fwd_id.vsi_list_id = vsi_list_id;
		tmp_fltr.fltr_act = ICE_FWD_TO_VSI_LIST;
		/* Update the previous switch rule to a new VSI list which
		 * includes current VSI that is requested
		 */
		status = ice_update_pkt_fwd_rule(hw, &tmp_fltr);
		if (status)
			goto exit;

		/* before overriding VSI list map info. decrement ref_cnt of
		 * previous VSI list
		 */
		v_list_itr->vsi_list_info->ref_cnt--;

		/* now update to newly created list */
		v_list_itr->fltr_info.fwd_id.vsi_list_id = vsi_list_id;
		v_list_itr->vsi_list_info =
			ice_create_vsi_list_map(hw, &vsi_handle_arr[0], 2,
						vsi_list_id);
		v_list_itr->vsi_count++;
	}

exit:
#if 0
	ice_release_lock(rule_lock);
#endif
	return status;
}

/**
 * ice_replay_vsi_fltr - Replay filters for requested VSI
 * @hw: pointer to the hardware structure
 * @pi: pointer to port information structure
 * @sw: pointer to switch info struct for which function replays filters
 * @vsi_handle: driver VSI handle
 * @recp_id: Recipe ID for which rules need to be replayed
 * @list_head: list for which filters need to be replayed
 *
 * Replays the filter of recipe recp_id for a VSI represented via vsi_handle.
 * It is required to pass valid VSI handle.
 */
enum ice_status
ice_replay_vsi_fltr(struct ice_hw *hw, struct ice_port_info *pi,
		    struct ice_switch_info *sw, uint16_t vsi_handle,
		    uint8_t recp_id,
		    struct ice_fltr_mgmt_list_head *list_head)
{
	struct ice_fltr_mgmt_list_entry *itr;
	enum ice_status status = ICE_SUCCESS;
	struct ice_sw_recipe *recp_list;
	uint16_t hw_vsi_id;

	if (TAILQ_EMPTY(list_head))
		return status;
	recp_list = &sw->recp_list[recp_id];
	hw_vsi_id = hw->vsi_ctx[vsi_handle]->vsi_num;

	TAILQ_FOREACH(itr, list_head, list_entry) {
		struct ice_fltr_list_entry f_entry;

		f_entry.fltr_info = itr->fltr_info;
		if (itr->vsi_count < 2 && recp_id != ICE_SW_LKUP_VLAN &&
		    itr->fltr_info.vsi_handle == vsi_handle) {
			/* update the src in case it is VSI num */
			if (f_entry.fltr_info.src_id == ICE_SRC_ID_VSI)
				f_entry.fltr_info.src = hw_vsi_id;
			status = ice_add_rule_internal(hw, recp_list,
						       pi->lport,
						       &f_entry);
			if (status != ICE_SUCCESS)
				goto end;
			continue;
		}
		if (!itr->vsi_list_info ||
		    !ice_is_bit_set(itr->vsi_list_info->vsi_map, vsi_handle))
			continue;
		/* Clearing it so that the logic can add it back */
		ice_clear_bit(vsi_handle, itr->vsi_list_info->vsi_map);
		f_entry.fltr_info.vsi_handle = vsi_handle;
		f_entry.fltr_info.fltr_act = ICE_FWD_TO_VSI;
		/* update the src in case it is VSI num */
		if (f_entry.fltr_info.src_id == ICE_SRC_ID_VSI)
			f_entry.fltr_info.src = hw_vsi_id;
		if (recp_id == ICE_SW_LKUP_VLAN)
			status = ice_add_vlan_internal(hw, recp_list, &f_entry);
		else
			status = ice_add_rule_internal(hw, recp_list,
						       pi->lport,
						       &f_entry);
		if (status != ICE_SUCCESS)
			goto end;
	}
end:
	return status;
}

/**
 * ice_replay_vsi_all_fltr - replay all filters stored in bookkeeping lists
 * @hw: pointer to the hardware structure
 * @pi: pointer to port information structure
 * @vsi_handle: driver VSI handle
 *
 * Replays filters for requested VSI via vsi_handle.
 */
enum ice_status
ice_replay_vsi_all_fltr(struct ice_hw *hw, struct ice_port_info *pi,
			uint16_t vsi_handle)
{
	struct ice_switch_info *sw = NULL;
	enum ice_status status = ICE_SUCCESS;
	uint8_t i;

	sw = hw->switch_info;

	/* Update the recipes that were created */
	for (i = 0; i < ICE_MAX_NUM_RECIPES; i++) {
		struct ice_fltr_mgmt_list_head *head;

		head = &sw->recp_list[i].filt_replay_rules;
		if (!sw->recp_list[i].adv_rule)
			status = ice_replay_vsi_fltr(hw, pi, sw, vsi_handle, i,
						     head);
		if (status != ICE_SUCCESS)
			return status;
	}

	return ICE_SUCCESS;
}

/**
 * ice_sched_replay_vsi_agg - replay aggregator & VSI to aggregator node(s)
 * @hw: pointer to the HW struct
 * @vsi_handle: software VSI handle
 *
 * This function replays aggregator node, VSI to aggregator type nodes, and
 * their node bandwidth information. This function needs to be called with
 * scheduler lock held.
 */
enum ice_status
ice_sched_replay_vsi_agg(struct ice_hw *hw, uint16_t vsi_handle)
{
#if 0
	ice_declare_bitmap(replay_bitmap, ICE_MAX_TRAFFIC_CLASS);
	struct ice_sched_agg_vsi_info *agg_vsi_info;
	struct ice_port_info *pi = hw->port_info;
	struct ice_sched_agg_info *agg_info;
	enum ice_status status;

	ice_zero_bitmap(replay_bitmap, ICE_MAX_TRAFFIC_CLASS);
	if (!ice_is_vsi_valid(hw, vsi_handle))
		return ICE_ERR_PARAM;
	agg_info = ice_get_vsi_agg_info(hw, vsi_handle);
	if (!agg_info)
		return ICE_SUCCESS; /* Not present in list - default Agg case */
	agg_vsi_info = ice_get_agg_vsi_info(agg_info, vsi_handle);
	if (!agg_vsi_info)
		return ICE_SUCCESS; /* Not present in list - default Agg case */
	ice_sched_get_ena_tc_bitmap(pi, agg_info->replay_tc_bitmap,
				    replay_bitmap);
	/* Replay aggregator node associated to vsi_handle */
	status = ice_sched_cfg_agg(hw->port_info, agg_info->agg_id,
				   ICE_AGG_TYPE_AGG, replay_bitmap);
	if (status)
		return status;
	/* Replay aggregator node BW (restore aggregator BW) */
	status = ice_sched_replay_agg_bw(hw, agg_info);
	if (status)
		return status;

	ice_zero_bitmap(replay_bitmap, ICE_MAX_TRAFFIC_CLASS);
	ice_sched_get_ena_tc_bitmap(pi, agg_vsi_info->replay_tc_bitmap,
				    replay_bitmap);
	/* Move this VSI (vsi_handle) to above aggregator */
	status = ice_sched_assoc_vsi_to_agg(pi, agg_info->agg_id, vsi_handle,
					    replay_bitmap);
	if (status)
		return status;
	/* Replay VSI BW (restore VSI BW) */
	return ice_sched_replay_vsi_bw(hw, vsi_handle,
				       agg_vsi_info->tc_bitmap);
#else
	return ICE_ERR_NOT_IMPL;
#endif
}

/**
 * ice_replay_vsi_agg - replay VSI to aggregator node
 * @hw: pointer to the HW struct
 * @vsi_handle: software VSI handle
 *
 * This function replays association of VSI to aggregator type nodes, and
 * node bandwidth information.
 */
enum ice_status
ice_replay_vsi_agg(struct ice_hw *hw, uint16_t vsi_handle)
{
#if 0
	struct ice_port_info *pi = hw->port_info;
#endif
	enum ice_status status;
#if 0
	ice_acquire_lock(&pi->sched_lock);
#endif
	status = ice_sched_replay_vsi_agg(hw, vsi_handle);
#if 0
	ice_release_lock(&pi->sched_lock);
#endif
	return status;
}
/**
 * ice_replay_vsi - replay VSI configuration
 * @hw: pointer to the HW struct
 * @vsi_handle: driver VSI handle
 *
 * Restore all VSI configuration after reset. It is required to call this
 * function with main VSI first.
 */
enum ice_status
ice_replay_vsi(struct ice_hw *hw, uint16_t vsi_handle)
{
	struct ice_switch_info *sw = hw->switch_info;
	struct ice_port_info *pi = hw->port_info;
	enum ice_status status;

	if (!ice_is_vsi_valid(hw, vsi_handle))
		return ICE_ERR_PARAM;

	/* Replay pre-initialization if there is any */
	if (ice_is_main_vsi(hw, vsi_handle)) {
		status = ice_replay_pre_init(hw, sw);
		if (status)
			return status;
	}
	/* Replay per VSI all RSS configurations */
	status = ice_replay_rss_cfg(hw, vsi_handle);
	if (status)
		return status;
	/* Replay per VSI all filters */
	status = ice_replay_vsi_all_fltr(hw, pi, vsi_handle);
	if (!status)
		status = ice_replay_vsi_agg(hw, vsi_handle);
	return status;
}

/**
 * ice_replay_all_vsi_cfg - Replace configuration for all VSIs after reset
 * @sc: the device softc
 *
 * Replace the configuration for each VSI, and then cleanup replay
 * information. Called after a hardware reset in order to reconfigure the
 * active VSIs.
 */
int
ice_replay_all_vsi_cfg(struct ice_softc *sc)
{
	struct ice_hw *hw = &sc->hw;
	enum ice_status status;
	int i;

	for (i = 0 ; i < sc->num_available_vsi; i++) {
		struct ice_vsi *vsi = sc->all_vsi[i];

		if (!vsi)
			continue;

		status = ice_replay_vsi(hw, vsi->idx);
		if (status) {
			printf("%s: Failed to replay VSI %d, err %s "
			    "aq_err %s\n", sc->sc_dev.dv_xname,
			    vsi->idx, ice_status_str(status),
			    ice_aq_str(hw->adminq.sq_last_status));
			return (EIO);
		}
	}

	/* Cleanup replay filters after successful reconfiguration */
	ice_replay_post(hw);
	return (0);
}

/**
 * ice_aq_set_health_status_config - Configure FW health events
 * @hw: pointer to the HW struct
 * @event_source: type of diagnostic events to enable
 * @cd: pointer to command details structure or NULL
 *
 * Configure the health status event types that the firmware will send to this
 * PF. The supported event types are: PF-specific, all PFs, and global
 */
enum ice_status
ice_aq_set_health_status_config(struct ice_hw *hw, uint8_t event_source,
				struct ice_sq_cd *cd)
{
	struct ice_aqc_set_health_status_config *cmd;
	struct ice_aq_desc desc;

	cmd = &desc.params.set_health_status_config;

	ice_fill_dflt_direct_cmd_desc(&desc,
				      ice_aqc_opc_set_health_status_config);

	cmd->event_source = event_source;

	return ice_aq_send_cmd(hw, &desc, NULL, 0, cd);
}

/**
 * ice_init_health_events - Enable FW health event reporting
 * @sc: device softc
 *
 * Will try to enable firmware health event reporting, but shouldn't
 * cause any grief (to the caller) if this fails.
 */
void
ice_init_health_events(struct ice_softc *sc)
{
	enum ice_status status;
	uint8_t health_mask;

	if (!ice_is_bit_set(sc->feat_cap, ICE_FEATURE_HEALTH_STATUS))
		return;

	health_mask = ICE_AQC_HEALTH_STATUS_SET_PF_SPECIFIC_MASK |
		      ICE_AQC_HEALTH_STATUS_SET_GLOBAL_MASK;

	status = ice_aq_set_health_status_config(&sc->hw, health_mask, NULL);
	if (status) {
		printf("%s: Failed to enable firmware health events, "
		    "err %s aq_err %s\n", sc->sc_dev.dv_xname,
		    ice_status_str(status),
		    ice_aq_str(sc->hw.adminq.sq_last_status));
	} else
		ice_set_bit(ICE_FEATURE_HEALTH_STATUS, sc->feat_en);
}

/**
 * ice_add_rx_lldp_filter - add ethertype filter for Rx LLDP frames
 * @sc: the device private structure
 *
 * Add a switch ethertype filter which forwards the LLDP frames to the main PF
 * VSI. Called when the fw_lldp_agent is disabled, to allow the LLDP frames to
 * be forwarded to the stack.
 */
void
ice_add_rx_lldp_filter(struct ice_softc *sc)
{
#if 0
	struct ice_list_head ethertype_list;
	struct ice_vsi *vsi = &sc->pf_vsi;
	struct ice_hw *hw = &sc->hw;
	device_t dev = sc->dev;
	enum ice_status status;
	int err;
	u16 vsi_num;

	/*
	 * If FW is new enough, use a direct AQ command to perform the filter
	 * addition.
	 */
	if (ice_fw_supports_lldp_fltr_ctrl(hw)) {
		vsi_num = ice_get_hw_vsi_num(hw, vsi->idx);
		status = ice_lldp_fltr_add_remove(hw, vsi_num, true);
		if (status) {
			device_printf(dev,
			    "Failed to add Rx LLDP filter, err %s aq_err %s\n",
			    ice_status_str(status),
			    ice_aq_str(hw->adminq.sq_last_status));
		} else
			ice_set_state(&sc->state,
			    ICE_STATE_LLDP_RX_FLTR_FROM_DRIVER);
		return;
	}

	INIT_LIST_HEAD(&ethertype_list);

	/* Forward Rx LLDP frames to the stack */
	err = ice_add_ethertype_to_list(vsi, &ethertype_list,
					ETHERTYPE_LLDP_FRAMES,
					ICE_FLTR_RX, ICE_FWD_TO_VSI);
	if (err) {
		device_printf(dev,
			      "Failed to add Rx LLDP filter, err %s\n",
			      ice_err_str(err));
		goto free_ethertype_list;
	}

	status = ice_add_eth_mac(hw, &ethertype_list);
	if (status && status != ICE_ERR_ALREADY_EXISTS) {
		device_printf(dev,
			      "Failed to add Rx LLDP filter, err %s aq_err %s\n",
			      ice_status_str(status),
			      ice_aq_str(hw->adminq.sq_last_status));
	} else {
		/*
		 * If status == ICE_ERR_ALREADY_EXISTS, we won't treat an
		 * already existing filter as an error case.
		 */
		ice_set_state(&sc->state, ICE_STATE_LLDP_RX_FLTR_FROM_DRIVER);
	}

free_ethertype_list:
	ice_free_fltr_list(&ethertype_list);
#else
	printf("%s: not implemented\n", __func__);
#endif
}

/**
 * ice_update_link_info - update status of the HW network link
 * @pi: port info structure of the interested logical port
 */
enum ice_status
ice_update_link_info(struct ice_port_info *pi)
{
	struct ice_link_status *li;
	enum ice_status status;

	if (!pi)
		return ICE_ERR_PARAM;

	li = &pi->phy.link_info;

	status = ice_aq_get_link_info(pi, true, NULL, NULL);
	if (status)
		return status;

	if (li->link_info & ICE_AQ_MEDIA_AVAILABLE) {
		struct ice_aqc_get_phy_caps_data *pcaps;
		struct ice_hw *hw;

		hw = pi->hw;
		pcaps = (struct ice_aqc_get_phy_caps_data *)
			ice_malloc(hw, sizeof(*pcaps));
		if (!pcaps)
			return ICE_ERR_NO_MEMORY;

		status = ice_aq_get_phy_caps(pi, false,
		    ICE_AQC_REPORT_TOPO_CAP_MEDIA, pcaps, NULL);

		if (status == ICE_SUCCESS)
			memcpy(li->module_type, &pcaps->module_type,
				   sizeof(li->module_type));

		ice_free(hw, pcaps);
	}

	return status;
}

/**
 * ice_get_link_status - get status of the HW network link
 * @pi: port information structure
 * @link_up: pointer to bool (true/false = linkup/linkdown)
 *
 * Variable link_up is true if link is up, false if link is down.
 * The variable link_up is invalid if status is non zero. As a
 * result of this call, link status reporting becomes enabled
 */
enum ice_status
ice_get_link_status(struct ice_port_info *pi, bool *link_up)
{
	struct ice_phy_info *phy_info;
	enum ice_status status = ICE_SUCCESS;

	if (!pi || !link_up)
		return ICE_ERR_PARAM;

	phy_info = &pi->phy;

	if (phy_info->get_link_info) {
		status = ice_update_link_info(pi);
		if (status) {
			DNPRINTF(ICE_DBG_LINK,
			    "%s: get link status error, status = %d\n",
			    __func__, status);
		}
	}

	*link_up = phy_info->link_info.link_info & ICE_AQ_LINK_UP;

	return status;
}

/**
 * ice_set_default_local_mib_settings - Set Local LLDP MIB to default settings
 * @sc: device softc structure
 *
 * Overwrites the driver's SW local LLDP MIB with default settings. This
 * ensures the driver has a valid MIB when it next uses the Set Local LLDP MIB
 * admin queue command.
 */
void
ice_set_default_local_mib_settings(struct ice_softc *sc)
{
	struct ice_dcbx_cfg *dcbcfg;
	struct ice_hw *hw = &sc->hw;
	struct ice_port_info *pi;
	uint8_t maxtcs, maxtcs_ets, old_pfc_mode;

	pi = hw->port_info;

	dcbcfg = &pi->qos_cfg.local_dcbx_cfg;

	maxtcs = hw->func_caps.common_cap.maxtc;
	/* This value is only 3 bits; 8 TCs maps to 0 */
	maxtcs_ets = maxtcs & ICE_IEEE_ETS_MAXTC_M;

	/* VLAN vs DSCP mode needs to be preserved */
	old_pfc_mode = dcbcfg->pfc_mode;

	/**
	 * Setup the default settings used by the driver for the Set Local
	 * LLDP MIB Admin Queue command (0x0A08). (1TC w/ 100% BW, ETS, no
	 * PFC, TSA=2).
	 */
	memset(dcbcfg, 0, sizeof(*dcbcfg));

	dcbcfg->etscfg.willing = 1;
	dcbcfg->etscfg.tcbwtable[0] = 100;
	dcbcfg->etscfg.maxtcs = maxtcs_ets;
	dcbcfg->etscfg.tsatable[0] = 2;

	dcbcfg->etsrec = dcbcfg->etscfg;
	dcbcfg->etsrec.willing = 0;

	dcbcfg->pfc.willing = 1;
	dcbcfg->pfc.pfccap = maxtcs;

	dcbcfg->pfc_mode = old_pfc_mode;
}

/**
 * ice_add_ieee_ets_common_tlv
 * @buf: Data buffer to be populated with ice_dcb_ets_cfg data
 * @ets_cfg: Container for ice_dcb_ets_cfg data
 *
 * Populate the TLV buffer with ice_dcb_ets_cfg data
 */
void
ice_add_ieee_ets_common_tlv(uint8_t *buf, struct ice_dcb_ets_cfg *ets_cfg)
{
	uint8_t priority0, priority1;
	uint8_t offset = 0;
	int i;

	/* Priority Assignment Table (4 octets)
	 * Octets:|    1    |    2    |    3    |    4    |
	 *        -----------------------------------------
	 *        |pri0|pri1|pri2|pri3|pri4|pri5|pri6|pri7|
	 *        -----------------------------------------
	 *   Bits:|7  4|3  0|7  4|3  0|7  4|3  0|7  4|3  0|
	 *        -----------------------------------------
	 */
	for (i = 0; i < ICE_MAX_TRAFFIC_CLASS / 2; i++) {
		priority0 = ets_cfg->prio_table[i * 2] & 0xF;
		priority1 = ets_cfg->prio_table[i * 2 + 1] & 0xF;
		buf[offset] = (priority0 << ICE_IEEE_ETS_PRIO_1_S) | priority1;
		offset++;
	}

	/* TC Bandwidth Table (8 octets)
	 * Octets:| 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 |
	 *        ---------------------------------
	 *        |tc0|tc1|tc2|tc3|tc4|tc5|tc6|tc7|
	 *        ---------------------------------
	 *
	 * TSA Assignment Table (8 octets)
	 * Octets:| 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 |
	 *        ---------------------------------
	 *        |tc0|tc1|tc2|tc3|tc4|tc5|tc6|tc7|
	 *        ---------------------------------
	 */
	ice_for_each_traffic_class(i) {
		buf[offset] = ets_cfg->tcbwtable[i];
		buf[ICE_MAX_TRAFFIC_CLASS + offset] = ets_cfg->tsatable[i];
		offset++;
	}
}

/**
 * ice_add_ieee_ets_tlv - Prepare ETS TLV in IEEE format
 * @tlv: Fill the ETS config data in IEEE format
 * @dcbcfg: Local store which holds the DCB Config
 *
 * Prepare IEEE 802.1Qaz ETS CFG TLV
 */
void
ice_add_ieee_ets_tlv(struct ice_lldp_org_tlv *tlv, struct ice_dcbx_cfg *dcbcfg)
{
	struct ice_dcb_ets_cfg *etscfg;
	uint8_t *buf = tlv->tlvinfo;
	uint8_t maxtcwilling = 0;
	uint32_t ouisubtype;
	uint16_t typelen;

	typelen = ((ICE_TLV_TYPE_ORG << ICE_LLDP_TLV_TYPE_S) |
		   ICE_IEEE_ETS_TLV_LEN);
	tlv->typelen = HTONS(typelen);

	ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) |
		      ICE_IEEE_SUBTYPE_ETS_CFG);
	tlv->ouisubtype = htonl(ouisubtype);

	/* First Octet post subtype
	 * --------------------------
	 * |will-|CBS  | Re-  | Max |
	 * |ing  |     |served| TCs |
	 * --------------------------
	 * |1bit | 1bit|3 bits|3bits|
	 */
	etscfg = &dcbcfg->etscfg;
	if (etscfg->willing)
		maxtcwilling = BIT(ICE_IEEE_ETS_WILLING_S);
	maxtcwilling |= etscfg->maxtcs & ICE_IEEE_ETS_MAXTC_M;
	buf[0] = maxtcwilling;

	/* Begin adding at Priority Assignment Table (offset 1 in buf) */
	ice_add_ieee_ets_common_tlv(&buf[1], etscfg);
}

/**
 * ice_add_ieee_etsrec_tlv - Prepare ETS Recommended TLV in IEEE format
 * @tlv: Fill ETS Recommended TLV in IEEE format
 * @dcbcfg: Local store which holds the DCB Config
 *
 * Prepare IEEE 802.1Qaz ETS REC TLV
 */
void
ice_add_ieee_etsrec_tlv(struct ice_lldp_org_tlv *tlv,
			struct ice_dcbx_cfg *dcbcfg)
{
	struct ice_dcb_ets_cfg *etsrec;
	uint8_t *buf = tlv->tlvinfo;
	uint32_t ouisubtype;
	uint16_t typelen;

	typelen = ((ICE_TLV_TYPE_ORG << ICE_LLDP_TLV_TYPE_S) |
		   ICE_IEEE_ETS_TLV_LEN);
	tlv->typelen = htons(typelen);

	ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) |
		      ICE_IEEE_SUBTYPE_ETS_REC);
	tlv->ouisubtype = HTONL(ouisubtype);

	etsrec = &dcbcfg->etsrec;

	/* First Octet is reserved */
	/* Begin adding at Priority Assignment Table (offset 1 in buf) */
	ice_add_ieee_ets_common_tlv(&buf[1], etsrec);
}

/**
 * ice_add_ieee_pfc_tlv - Prepare PFC TLV in IEEE format
 * @tlv: Fill PFC TLV in IEEE format
 * @dcbcfg: Local store which holds the PFC CFG data
 *
 * Prepare IEEE 802.1Qaz PFC CFG TLV
 */
void
ice_add_ieee_pfc_tlv(struct ice_lldp_org_tlv *tlv, struct ice_dcbx_cfg *dcbcfg)
{
	uint8_t *buf = tlv->tlvinfo;
	uint32_t ouisubtype;
	uint16_t typelen;

	typelen = ((ICE_TLV_TYPE_ORG << ICE_LLDP_TLV_TYPE_S) |
		   ICE_IEEE_PFC_TLV_LEN);
	tlv->typelen = htons(typelen);

	ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) |
		      ICE_IEEE_SUBTYPE_PFC_CFG);
	tlv->ouisubtype = htonl(ouisubtype);

	/* ----------------------------------------
	 * |will-|MBC  | Re-  | PFC |  PFC Enable  |
	 * |ing  |     |served| cap |              |
	 * -----------------------------------------
	 * |1bit | 1bit|2 bits|4bits| 1 octet      |
	 */
	if (dcbcfg->pfc.willing)
		buf[0] = BIT(ICE_IEEE_PFC_WILLING_S);

	if (dcbcfg->pfc.mbc)
		buf[0] |= BIT(ICE_IEEE_PFC_MBC_S);

	buf[0] |= dcbcfg->pfc.pfccap & 0xF;
	buf[1] = dcbcfg->pfc.pfcena;
}

/**
 * ice_add_ieee_app_pri_tlv -  Prepare APP TLV in IEEE format
 * @tlv: Fill APP TLV in IEEE format
 * @dcbcfg: Local store which holds the APP CFG data
 *
 * Prepare IEEE 802.1Qaz APP CFG TLV
 */
void
ice_add_ieee_app_pri_tlv(struct ice_lldp_org_tlv *tlv,
			 struct ice_dcbx_cfg *dcbcfg)
{
	uint16_t typelen, len, offset = 0;
	uint8_t priority, selector, i = 0;
	uint8_t *buf = tlv->tlvinfo;
	uint32_t ouisubtype;

	/* No APP TLVs then just return */
	if (dcbcfg->numapps == 0)
		return;
	ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) |
		      ICE_IEEE_SUBTYPE_APP_PRI);
	tlv->ouisubtype = HTONL(ouisubtype);

	/* Move offset to App Priority Table */
	offset++;
	/* Application Priority Table (3 octets)
	 * Octets:|         1          |    2    |    3    |
	 *        -----------------------------------------
	 *        |Priority|Rsrvd| Sel |    Protocol ID    |
	 *        -----------------------------------------
	 *   Bits:|23    21|20 19|18 16|15                0|
	 *        -----------------------------------------
	 */
	while (i < dcbcfg->numapps) {
		priority = dcbcfg->app[i].priority & 0x7;
		selector = dcbcfg->app[i].selector & 0x7;
		buf[offset] = (priority << ICE_IEEE_APP_PRIO_S) | selector;
		buf[offset + 1] = (dcbcfg->app[i].prot_id >> 0x8) & 0xFF;
		buf[offset + 2] = dcbcfg->app[i].prot_id & 0xFF;
		/* Move to next app */
		offset += 3;
		i++;
		if (i >= ICE_DCBX_MAX_APPS)
			break;
	}
	/* len includes size of ouisubtype + 1 reserved + 3*numapps */
	len = sizeof(tlv->ouisubtype) + 1 + (i * 3);
	typelen = ((ICE_TLV_TYPE_ORG << ICE_LLDP_TLV_TYPE_S) | (len & 0x1FF));
	tlv->typelen = HTONS(typelen);
}

/**
 * ice_add_dscp_up_tlv - Prepare DSCP to UP TLV
 * @tlv: location to build the TLV data
 * @dcbcfg: location of data to convert to TLV
 */
void
ice_add_dscp_up_tlv(struct ice_lldp_org_tlv *tlv, struct ice_dcbx_cfg *dcbcfg)
{
	uint8_t *buf = tlv->tlvinfo;
	uint32_t ouisubtype;
	uint16_t typelen;
	int i;

	typelen = ((ICE_TLV_TYPE_ORG << ICE_LLDP_TLV_TYPE_S) |
		   ICE_DSCP_UP_TLV_LEN);
	tlv->typelen = htons(typelen);

	ouisubtype = (uint32_t)((ICE_DSCP_OUI << ICE_LLDP_TLV_OUI_S) |
			   ICE_DSCP_SUBTYPE_DSCP2UP);
	tlv->ouisubtype = htonl(ouisubtype);

	/* bytes 0 - 63 - IPv4 DSCP2UP LUT */
	for (i = 0; i < ICE_DSCP_NUM_VAL; i++) {
		/* IPv4 mapping */
		buf[i] = dcbcfg->dscp_map[i];
		/* IPv6 mapping */
		buf[i + ICE_DSCP_IPV6_OFFSET] = dcbcfg->dscp_map[i];
	}

	/* byte 64 - IPv4 untagged traffic */
	buf[i] = 0;

	/* byte 144 - IPv6 untagged traffic */
	buf[i + ICE_DSCP_IPV6_OFFSET] = 0;
}

#define ICE_BYTES_PER_TC	8

/**
 * ice_add_dscp_enf_tlv - Prepare DSCP Enforcement TLV
 * @tlv: location to build the TLV data
 */
void
ice_add_dscp_enf_tlv(struct ice_lldp_org_tlv *tlv)
{
	uint8_t *buf = tlv->tlvinfo;
	uint32_t ouisubtype;
	uint16_t typelen;

	typelen = ((ICE_TLV_TYPE_ORG << ICE_LLDP_TLV_TYPE_S) |
		   ICE_DSCP_ENF_TLV_LEN);
	tlv->typelen = htons(typelen);

	ouisubtype = (uint32_t)((ICE_DSCP_OUI << ICE_LLDP_TLV_OUI_S) |
			   ICE_DSCP_SUBTYPE_ENFORCE);
	tlv->ouisubtype = htonl(ouisubtype);

	/* Allow all DSCP values to be valid for all TC's (IPv4 and IPv6) */
	memset(buf, 0, 2 * (ICE_MAX_TRAFFIC_CLASS * ICE_BYTES_PER_TC));
}

/**
 * ice_add_dscp_tc_bw_tlv - Prepare DSCP BW for TC TLV
 * @tlv: location to build the TLV data
 * @dcbcfg: location of the data to convert to TLV
 */
void
ice_add_dscp_tc_bw_tlv(struct ice_lldp_org_tlv *tlv,
		       struct ice_dcbx_cfg *dcbcfg)
{
	struct ice_dcb_ets_cfg *etscfg;
	uint8_t *buf = tlv->tlvinfo;
	uint32_t ouisubtype;
	uint8_t offset = 0;
	uint16_t typelen;
	int i;

	typelen = ((ICE_TLV_TYPE_ORG << ICE_LLDP_TLV_TYPE_S) |
		   ICE_DSCP_TC_BW_TLV_LEN);
	tlv->typelen = htons(typelen);

	ouisubtype = (uint32_t)((ICE_DSCP_OUI << ICE_LLDP_TLV_OUI_S) |
			   ICE_DSCP_SUBTYPE_TCBW);
	tlv->ouisubtype = htonl(ouisubtype);

	/* First Octect after subtype
	 * ----------------------------
	 * | RSV | CBS | RSV | Max TCs |
	 * | 1b  | 1b  | 3b  | 3b      |
	 * ----------------------------
	 */
	etscfg = &dcbcfg->etscfg;
	buf[0] = etscfg->maxtcs & ICE_IEEE_ETS_MAXTC_M;

	/* bytes 1 - 4 reserved */
	offset = 5;

	/* TC BW table
	 * bytes 0 - 7 for TC 0 - 7
	 *
	 * TSA Assignment table
	 * bytes 8 - 15 for TC 0 - 7
	 */
	for (i = 0; i < ICE_MAX_TRAFFIC_CLASS; i++) {
		buf[offset] = etscfg->tcbwtable[i];
		buf[offset + ICE_MAX_TRAFFIC_CLASS] = etscfg->tsatable[i];
		offset++;
	}
}

/**
 * ice_add_dscp_pfc_tlv - Prepare DSCP PFC TLV
 * @tlv: Fill PFC TLV in IEEE format
 * @dcbcfg: Local store which holds the PFC CFG data
 */
void
ice_add_dscp_pfc_tlv(struct ice_lldp_org_tlv *tlv, struct ice_dcbx_cfg *dcbcfg)
{
	uint8_t *buf = tlv->tlvinfo;
	uint32_t ouisubtype;
	uint16_t typelen;

	typelen = ((ICE_TLV_TYPE_ORG << ICE_LLDP_TLV_TYPE_S) |
		   ICE_DSCP_PFC_TLV_LEN);
	tlv->typelen = htons(typelen);

	ouisubtype = (uint32_t)((ICE_DSCP_OUI << ICE_LLDP_TLV_OUI_S) |
			   ICE_DSCP_SUBTYPE_PFC);
	tlv->ouisubtype = HTONL(ouisubtype);

	buf[0] = dcbcfg->pfc.pfccap & 0xF;
	buf[1] = dcbcfg->pfc.pfcena;
}

/**
 * ice_add_dcb_tlv - Add all IEEE or DSCP TLVs
 * @tlv: Fill TLV data in IEEE format
 * @dcbcfg: Local store which holds the DCB Config
 * @tlvid: Type of IEEE TLV
 *
 * Add tlv information
 */
void
ice_add_dcb_tlv(struct ice_lldp_org_tlv *tlv, struct ice_dcbx_cfg *dcbcfg,
		uint16_t tlvid)
{
	if (dcbcfg->pfc_mode == ICE_QOS_MODE_VLAN) {
		switch (tlvid) {
		case ICE_IEEE_TLV_ID_ETS_CFG:
			ice_add_ieee_ets_tlv(tlv, dcbcfg);
			break;
		case ICE_IEEE_TLV_ID_ETS_REC:
			ice_add_ieee_etsrec_tlv(tlv, dcbcfg);
			break;
		case ICE_IEEE_TLV_ID_PFC_CFG:
			ice_add_ieee_pfc_tlv(tlv, dcbcfg);
			break;
		case ICE_IEEE_TLV_ID_APP_PRI:
			ice_add_ieee_app_pri_tlv(tlv, dcbcfg);
			break;
		default:
			break;
		}
	} else {
		/* pfc_mode == ICE_QOS_MODE_DSCP */
		switch (tlvid) {
		case ICE_TLV_ID_DSCP_UP:
			ice_add_dscp_up_tlv(tlv, dcbcfg);
			break;
		case ICE_TLV_ID_DSCP_ENF:
			ice_add_dscp_enf_tlv(tlv);
			break;
		case ICE_TLV_ID_DSCP_TC_BW:
			ice_add_dscp_tc_bw_tlv(tlv, dcbcfg);
			break;
		case ICE_TLV_ID_DSCP_TO_PFC:
			ice_add_dscp_pfc_tlv(tlv, dcbcfg);
			break;
		default:
			break;
		}
	}
}
/**
 * ice_dcb_cfg_to_lldp - Convert DCB configuration to MIB format
 * @lldpmib: pointer to the HW struct
 * @miblen: length of LLDP MIB
 * @dcbcfg: Local store which holds the DCB Config
 *
 * Convert the DCB configuration to MIB format
 */
void
ice_dcb_cfg_to_lldp(uint8_t *lldpmib, uint16_t *miblen,
    struct ice_dcbx_cfg *dcbcfg)
{
	uint16_t len, offset = 0, tlvid = ICE_TLV_ID_START;
	struct ice_lldp_org_tlv *tlv;
	uint16_t typelen;

	tlv = (struct ice_lldp_org_tlv *)lldpmib;
	while (1) {
		ice_add_dcb_tlv(tlv, dcbcfg, tlvid++);
		typelen = ntohs(tlv->typelen);
		len = (typelen & ICE_LLDP_TLV_LEN_M) >> ICE_LLDP_TLV_LEN_S;
		if (len)
			offset += len + 2;
		/* END TLV or beyond LLDPDU size */
		if (tlvid >= ICE_TLV_ID_END_OF_LLDPPDU ||
		    offset > ICE_LLDPDU_SIZE)
			break;
		/* Move to next TLV */
		if (len)
			tlv = (struct ice_lldp_org_tlv *)
				((char *)tlv + sizeof(tlv->typelen) + len);
	}
	*miblen = offset;
}

/**
 * ice_aq_set_lldp_mib - Set the LLDP MIB
 * @hw: pointer to the HW struct
 * @mib_type: Local, Remote or both Local and Remote MIBs
 * @buf: pointer to the caller-supplied buffer to store the MIB block
 * @buf_size: size of the buffer (in bytes)
 * @cd: pointer to command details structure or NULL
 *
 * Set the LLDP MIB. (0x0A08)
 */
enum ice_status
ice_aq_set_lldp_mib(struct ice_hw *hw, uint8_t mib_type, void *buf,
    uint16_t buf_size, struct ice_sq_cd *cd)
{
	struct ice_aqc_lldp_set_local_mib *cmd;
	struct ice_aq_desc desc;

	cmd = &desc.params.lldp_set_mib;

	if (buf_size == 0 || !buf)
		return ICE_ERR_PARAM;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_lldp_set_local_mib);

	desc.flags |= htole16((uint16_t)ICE_AQ_FLAG_RD);
	desc.datalen = htole16(buf_size);

	cmd->type = mib_type;
	cmd->length = htole16(buf_size);

	return ice_aq_send_cmd(hw, &desc, buf, buf_size, cd);
}

/**
 * ice_set_dcb_cfg - Set the local LLDP MIB to FW
 * @pi: port information structure
 *
 * Set DCB configuration to the Firmware
 */
enum ice_status
ice_set_dcb_cfg(struct ice_port_info *pi)
{
	uint8_t mib_type, *lldpmib = NULL;
	struct ice_dcbx_cfg *dcbcfg;
	enum ice_status ret;
	struct ice_hw *hw;
	uint16_t miblen;

	if (!pi)
		return ICE_ERR_PARAM;

	hw = pi->hw;

	/* update the HW local config */
	dcbcfg = &pi->qos_cfg.local_dcbx_cfg;
	/* Allocate the LLDPDU */
	lldpmib = (uint8_t *)ice_malloc(hw, ICE_LLDPDU_SIZE);
	if (!lldpmib)
		return ICE_ERR_NO_MEMORY;

	mib_type = SET_LOCAL_MIB_TYPE_LOCAL_MIB;
	if (dcbcfg->app_mode == ICE_DCBX_APPS_NON_WILLING)
		mib_type |= SET_LOCAL_MIB_TYPE_CEE_NON_WILLING;

	ice_dcb_cfg_to_lldp(lldpmib, &miblen, dcbcfg);
	ret = ice_aq_set_lldp_mib(hw, mib_type, (void *)lldpmib, miblen,
				  NULL);

	ice_free(hw, lldpmib);

	return ret;
}

/**
 * ice_set_default_local_lldp_mib - Possibly apply local LLDP MIB to FW
 * @sc: device softc structure
 *
 * This function needs to be called after link up; it makes sure the FW has
 * certain PFC/DCB settings. In certain configurations this will re-apply a
 * default local LLDP MIB configuration; this is intended to workaround a FW
 * behavior where these settings seem to be cleared on link up.
 */
void
ice_set_default_local_lldp_mib(struct ice_softc *sc)
{
	struct ice_hw *hw = &sc->hw;
	struct ice_port_info *pi;
	enum ice_status status;

	/* Set Local MIB can disrupt flow control settings for
	 * non-DCB-supported devices.
	 */
	if (!ice_is_bit_set(sc->feat_en, ICE_FEATURE_DCB))
		return;

	pi = hw->port_info;

	/* Don't overwrite a custom SW configuration */
	if (!pi->qos_cfg.is_sw_lldp &&
	    !ice_test_state(&sc->state, ICE_STATE_MULTIPLE_TCS))
		ice_set_default_local_mib_settings(sc);

	status = ice_set_dcb_cfg(pi);
	if (status) {
		printf("%s: Error setting Local LLDP MIB: %s aq_err %s\n",
		    sc->sc_dev.dv_xname,
		    ice_status_str(status),
		    ice_aq_str(hw->adminq.sq_last_status));
	}
}

/**
 * ice_aq_speed_to_rate - Convert AdminQ speed enum to baudrate
 * @pi: port info data
 *
 * Returns the baudrate value for the current link speed of a given port.
 */
uint64_t
ice_aq_speed_to_rate(struct ice_port_info *pi)
{
	switch (pi->phy.link_info.link_speed) {
	case ICE_AQ_LINK_SPEED_100GB:
		return IF_Gbps(100);
	case ICE_AQ_LINK_SPEED_50GB:
		return IF_Gbps(50);
	case ICE_AQ_LINK_SPEED_40GB:
		return IF_Gbps(40);
	case ICE_AQ_LINK_SPEED_25GB:
		return IF_Gbps(25);
	case ICE_AQ_LINK_SPEED_10GB:
		return IF_Gbps(10);
	case ICE_AQ_LINK_SPEED_5GB:
		return IF_Gbps(5);
	case ICE_AQ_LINK_SPEED_2500MB:
		return IF_Mbps(2500);
	case ICE_AQ_LINK_SPEED_1000MB:
		return IF_Mbps(1000);
	case ICE_AQ_LINK_SPEED_100MB:
		return IF_Mbps(100);
	case ICE_AQ_LINK_SPEED_10MB:
		return IF_Mbps(10);
	case ICE_AQ_LINK_SPEED_UNKNOWN:
	default:
		/* return 0 if we don't know the link speed */
		return 0;
	}
}

/**
 * ice_update_link_status - notify OS of link state change
 * @sc: device private softc structure
 * @update_media: true if we should update media even if link didn't change
 *
 * Called to notify iflib core of link status changes. Should be called once
 * during attach_post, and whenever link status changes during runtime.
 *
 * This call only updates the currently supported media types if the link
 * status changed, or if update_media is set to true.
 */
void
ice_update_link_status(struct ice_softc *sc, bool update_media)
{
	struct ifnet *ifp = &sc->sc_ac.ac_if;
	struct ice_hw *hw = &sc->hw;
	enum ice_status status;

	/* Never report link up when in recovery mode */
	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
		return;

	/* Report link status to iflib only once each time it changes */
	if (!ice_testandset_state(&sc->state, ICE_STATE_LINK_STATUS_REPORTED)) {
		if (sc->link_up) { /* link is up */
			uint64_t baudrate;

			baudrate = ice_aq_speed_to_rate(sc->hw.port_info);
			if (!(hw->port_info->phy.link_info_old.link_info &
			    ICE_AQ_LINK_UP))
				ice_set_default_local_lldp_mib(sc);

			ifp->if_baudrate = baudrate;
			ifp->if_link_state = LINK_STATE_UP;
			if_link_state_change(ifp);
#if 0
			ice_rdma_link_change(sc, LINK_STATE_UP, baudrate);
			ice_link_up_msg(sc);
#endif
		} else { /* link is down */
			ifp->if_baudrate = 0;
			ifp->if_link_state = LINK_STATE_DOWN;
			if_link_state_change(ifp);
#if 0
			ice_rdma_link_change(sc, LINK_STATE_DOWN, 0);
#endif
		}
		update_media = true;
	}

	/* Update the supported media types */
	if (update_media &&
	    !ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) {
		status = ice_add_media_types(sc, &sc->media);
		if (status)
			printf("%s: Error adding device media types: "
			    "%s aq_err %s\n", sc->sc_dev.dv_xname,
			    ice_status_str(status),
			    ice_aq_str(hw->adminq.sq_last_status));
	}
}

/**
 * ice_rebuild - Rebuild driver state post reset
 * @sc: The device private softc
 *
 * Restore driver state after a reset occurred. Restart the controlqs, setup
 * the hardware port, and re-enable the VSIs.
 */
void
ice_rebuild(struct ice_softc *sc)
{
	struct ice_hw *hw = &sc->hw;
	enum ice_status status;
	int err;

	sc->rebuild_ticks = ticks;

	/* If we're rebuilding, then a reset has succeeded. */
	ice_clear_state(&sc->state, ICE_STATE_RESET_FAILED);

	/*
	 * If the firmware is in recovery mode, only restore the limited
	 * functionality supported by recovery mode.
	 */
	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) {
		ice_rebuild_recovery_mode(sc);
		return;
	}
#if 0
	/* enable PCIe bus master */
	pci_enable_busmaster(dev);
#endif
	status = ice_init_all_ctrlq(hw);
	if (status) {
		printf("%s: failed to re-init control queues, err %s\n",
		    sc->sc_dev.dv_xname, ice_status_str(status));
		goto err_shutdown_ctrlq;
	}

	/* Query the allocated resources for Tx scheduler */
	status = ice_sched_query_res_alloc(hw);
	if (status) {
		printf("%s: Failed to query scheduler resources, "
		    "err %s aq_err %s\n", sc->sc_dev.dv_xname,
		    ice_status_str(status),
		    ice_aq_str(hw->adminq.sq_last_status));
		goto err_shutdown_ctrlq;
	}

	/* Re-enable FW logging. Keep going even if this fails */
	status = ice_fwlog_set(hw, &hw->fwlog_cfg);
	if (!status) {
		/*
		 * We should have the most updated cached copy of the
		 * configuration, regardless of whether we're rebuilding
		 * or not.  So we'll simply check to see if logging was
		 * enabled pre-rebuild.
		 */
		if (hw->fwlog_cfg.options & ICE_FWLOG_OPTION_IS_REGISTERED) {
			status = ice_fwlog_register(hw);
			if (status)
				printf("%s: failed to re-register fw logging, "
				    "err %s aq_err %s\n",
				   sc->sc_dev.dv_xname,
				   ice_status_str(status),
				   ice_aq_str(hw->adminq.sq_last_status));
		}
	} else {
		printf("%s: failed to rebuild fw logging configuration, "
		    "err %s aq_err %s\n", sc->sc_dev.dv_xname,
		   ice_status_str(status),
		   ice_aq_str(hw->adminq.sq_last_status));
	}
	err = ice_send_version(sc);
	if (err)
		goto err_shutdown_ctrlq;

	err = ice_init_link_events(sc);
	if (err) {
		printf("%s: ice_init_link_events failed: %d\n",
		    sc->sc_dev.dv_xname, err);
		goto err_shutdown_ctrlq;
	}

	status = ice_clear_pf_cfg(hw);
	if (status) {
		printf("%s: failed to clear PF configuration, err %s\n",
		    sc->sc_dev.dv_xname, ice_status_str(status));
		goto err_shutdown_ctrlq;
	}

	ice_clean_all_vsi_rss_cfg(sc);

	ice_clear_pxe_mode(hw);

	status = ice_get_caps(hw);
	if (status) {
		printf("%s: failed to get capabilities, err %s\n",
		    sc->sc_dev.dv_xname, ice_status_str(status));
		goto err_shutdown_ctrlq;
	}

	status = ice_sched_init_port(hw->port_info);
	if (status) {
		printf("%s: failed to initialize port, err %s\n",
		     sc->sc_dev.dv_xname, ice_status_str(status));
		goto err_sched_cleanup;
	}

	/* If we previously loaded the package, it needs to be reloaded now */
	if (!ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE)) {
#if 0
		enum ice_ddp_state pkg_state;

		pkg_state = ice_init_pkg(hw, hw->pkg_copy, hw->pkg_size);
		if (!ice_is_init_pkg_successful(pkg_state)) {
			ice_log_pkg_init(sc, pkg_state);
			ice_transition_safe_mode(sc);
		}
#endif
	}

	ice_reset_pf_stats(sc);

	err = ice_rebuild_pf_vsi_qmap(sc);
	if (err) {
		printf("%s: Unable to re-assign main VSI queues, err %d\n",
		    sc->sc_dev.dv_xname, err);
		goto err_sched_cleanup;
	}
	err = ice_initialize_vsi(&sc->pf_vsi);
	if (err) {
		printf("%s: Unable to re-initialize Main VSI, err %d\n",
		    sc->sc_dev.dv_xname, err);
		goto err_release_queue_allocations;
	}

	/* Replay all VSI configuration */
	err = ice_replay_all_vsi_cfg(sc);
	if (err)
		goto err_deinit_pf_vsi;

	/* Re-enable FW health event reporting */
	ice_init_health_events(sc);
#if 0
	/* Reconfigure the main PF VSI for RSS */
	err = ice_config_rss(&sc->pf_vsi);
	if (err) {
		device_printf(sc->dev,
			      "Unable to reconfigure RSS for the main VSI, err %s\n",
			      ice_err_str(err));
		goto err_deinit_pf_vsi;
	}
#endif
	if (hw->port_info->qos_cfg.is_sw_lldp)
		ice_add_rx_lldp_filter(sc);

	/* Refresh link status */
	ice_clear_state(&sc->state, ICE_STATE_LINK_STATUS_REPORTED);
	sc->hw.port_info->phy.get_link_info = true;
	ice_get_link_status(sc->hw.port_info, &sc->link_up);
	ice_update_link_status(sc, true);

	/* RDMA interface will be restarted by the stack re-init */

	/* Configure interrupt causes for the administrative interrupt */
	ice_configure_misc_interrupts(sc);

	/* Enable ITR 0 right away, so that we can handle admin interrupts */
	ice_enable_intr(&sc->hw, 0);

	/* Rebuild is finished. We're no longer prepared to reset */
	ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET);
#if 0
	/* Reconfigure the subinterface */
	if (sc->mirr_if) {
		err = ice_subif_rebuild(sc);
		if (err)
			goto err_deinit_pf_vsi;
	}
#endif
	printf("%s: device rebuild successful\n", sc->sc_dev.dv_xname);

	/* In order to completely restore device functionality, the iflib core
	 * needs to be reset. We need to request an iflib reset. Additionally,
	 * because the state of IFC_DO_RESET is cached within task_fn_admin in
	 * the iflib core, we also want re-run the admin task so that iflib
	 * resets immediately instead of waiting for the next interrupt.
	 * If LLDP is enabled we need to reconfig DCB to properly reinit all TC
	 * queues, not only 0. It contains ice_request_stack_reinit as well.
	 */
#if 0
	if (hw->port_info->qos_cfg.is_sw_lldp)
		ice_request_stack_reinit(sc);
	else
		ice_do_dcb_reconfig(sc, false);
#endif
	return;

err_deinit_pf_vsi:
	ice_deinit_vsi(&sc->pf_vsi);
err_release_queue_allocations:
	ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap,
				    sc->pf_vsi.num_tx_queues);
	ice_resmgr_release_map(&sc->rx_qmgr, sc->pf_vsi.rx_qmap,
				    sc->pf_vsi.num_rx_queues);
err_sched_cleanup:
	ice_sched_cleanup_all(hw);
err_shutdown_ctrlq:
	ice_shutdown_all_ctrlq(hw, false);
	ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET);
	ice_set_state(&sc->state, ICE_STATE_RESET_FAILED);
	printf("%s: driver reset failed\n", sc->sc_dev.dv_xname);
}

/**
 * ice_handle_reset_event - Handle reset events triggered by OICR
 * @sc: The device private softc
 *
 * Handle reset events triggered by an OICR notification. This includes CORER,
 * GLOBR, and EMPR resets triggered by software on this or any other PF or by
 * firmware.
 *
 * @pre assumes the iflib context lock is held, and will unlock it while
 * waiting for the hardware to finish reset.
 */
void
ice_handle_reset_event(struct ice_softc *sc)
{
	struct ice_hw *hw = &sc->hw;
	enum ice_status status;

	/* When a CORER, GLOBR, or EMPR is about to happen, the hardware will
	 * trigger an OICR interrupt. Our OICR handler will determine when
	 * this occurs and set the ICE_STATE_RESET_OICR_RECV bit as
	 * appropriate.
	 */
	if (!ice_testandclear_state(&sc->state, ICE_STATE_RESET_OICR_RECV))
		return;

	ice_prepare_for_reset(sc);

	/*
	 * Release the iflib context lock and wait for the device to finish
	 * resetting.
	 */
#if 0
	IFLIB_CTX_UNLOCK(sc);
#endif
	status = ice_check_reset(hw);
#if 0
	IFLIB_CTX_LOCK(sc);
#endif
	if (status) {
		printf("%s: Device never came out of reset, err %s\n",
		    sc->sc_dev.dv_xname, ice_status_str(status));
		ice_set_state(&sc->state, ICE_STATE_RESET_FAILED);
		return;
	}

	/* We're done with the reset, so we can rebuild driver state */
	sc->hw.reset_ongoing = false;
	ice_rebuild(sc);

	/* In the unlikely event that a PF reset request occurs at the same
	 * time as a global reset, clear the request now. This avoids
	 * resetting a second time right after we reset due to a global event.
	 */
	if (ice_testandclear_state(&sc->state, ICE_STATE_RESET_PFR_REQ))
		printf("%s: Ignoring PFR request that occurred while a "
		    "reset was ongoing\n", sc->sc_dev.dv_xname);
}

/**
 * ice_handle_pf_reset_request - Initiate PF reset requested by software
 * @sc: The device private softc
 *
 * Initiate a PF reset requested by software. We handle this in the admin task
 * so that only one thread actually handles driver preparation and cleanup,
 * rather than having multiple threads possibly attempt to run this code
 * simultaneously.
 *
 * @pre assumes the iflib context lock is held and will unlock it while
 * waiting for the PF reset to complete.
 */
void
ice_handle_pf_reset_request(struct ice_softc *sc)
{
	struct ice_hw *hw = &sc->hw;
	enum ice_status status;

	/* Check for PF reset requests */
	if (!ice_testandclear_state(&sc->state, ICE_STATE_RESET_PFR_REQ))
		return;

	/* Make sure we're prepared for reset */
	ice_prepare_for_reset(sc);

	/*
	 * Release the iflib context lock and wait for the device to finish
	 * resetting.
	 */
#if 0
	IFLIB_CTX_UNLOCK(sc);
#endif
	status = ice_reset(hw, ICE_RESET_PFR);
#if 0
	IFLIB_CTX_LOCK(sc);
#endif
	if (status) {
		printf("%s: device PF reset failed, err %s\n",
		    sc->sc_dev.dv_xname, ice_status_str(status));
		ice_set_state(&sc->state, ICE_STATE_RESET_FAILED);
		return;
	}
#if 0
	sc->soft_stats.pfr_count++;
#endif
	ice_rebuild(sc);
}

/**
 * ice_handle_mdd_event - Handle possibly malicious events
 * @sc: the device softc
 *
 * Called by the admin task if an MDD detection interrupt is triggered.
 * Identifies possibly malicious events coming from VFs. Also triggers for
 * similar incorrect behavior from the PF as well.
 */
void
ice_handle_mdd_event(struct ice_softc *sc)
{
	struct ice_hw *hw = &sc->hw;
	bool mdd_detected = false, request_reinit = false;
	uint32_t reg;

	if (!ice_testandclear_state(&sc->state, ICE_STATE_MDD_PENDING))
		return;

	reg = ICE_READ(hw, GL_MDET_TX_TCLAN);
	if (reg & GL_MDET_TX_TCLAN_VALID_M) {
		uint8_t pf_num  = (reg & GL_MDET_TX_TCLAN_PF_NUM_M) >>
		    GL_MDET_TX_TCLAN_PF_NUM_S;
		uint16_t vf_num = (reg & GL_MDET_TX_TCLAN_VF_NUM_M) >>
		    GL_MDET_TX_TCLAN_VF_NUM_S;
		uint8_t event   = (reg & GL_MDET_TX_TCLAN_MAL_TYPE_M) >>
		    GL_MDET_TX_TCLAN_MAL_TYPE_S;
		uint16_t queue  = (reg & GL_MDET_TX_TCLAN_QNUM_M) >>
		    GL_MDET_TX_TCLAN_QNUM_S;

		printf("%s: Malicious Driver Detection Tx Descriptor "
		    "check event '%s' on Tx queue %u PF# %u VF# %u\n",
		    sc->sc_dev.dv_xname, ice_mdd_tx_tclan_str(event),
		    queue, pf_num, vf_num);

		/* Only clear this event if it matches this PF, that way other
		 * PFs can read the event and determine VF and queue number.
		 */
		if (pf_num == hw->pf_id)
			ICE_WRITE(hw, GL_MDET_TX_TCLAN, 0xffffffff);

		mdd_detected = true;
	}

	/* Determine what triggered the MDD event */
	reg = ICE_READ(hw, GL_MDET_TX_PQM);
	if (reg & GL_MDET_TX_PQM_VALID_M) {
		uint8_t pf_num  = (reg & GL_MDET_TX_PQM_PF_NUM_M) >>
		    GL_MDET_TX_PQM_PF_NUM_S;
		uint16_t vf_num = (reg & GL_MDET_TX_PQM_VF_NUM_M) >>
		    GL_MDET_TX_PQM_VF_NUM_S;
		uint8_t event   = (reg & GL_MDET_TX_PQM_MAL_TYPE_M) >>
		    GL_MDET_TX_PQM_MAL_TYPE_S;
		uint16_t queue  = (reg & GL_MDET_TX_PQM_QNUM_M) >>
		    GL_MDET_TX_PQM_QNUM_S;

		printf("%s: Malicious Driver Detection Tx Quanta check "
		    "event '%s' on Tx queue %u PF# %u VF# %u\n",
		    sc->sc_dev.dv_xname, ice_mdd_tx_pqm_str(event), queue,
		    pf_num, vf_num);

		/* Only clear this event if it matches this PF, that way other
		 * PFs can read the event and determine VF and queue number.
		 */
		if (pf_num == hw->pf_id)
			ICE_WRITE(hw, GL_MDET_TX_PQM, 0xffffffff);

		mdd_detected = true;
	}

	reg = ICE_READ(hw, GL_MDET_RX);
	if (reg & GL_MDET_RX_VALID_M) {
		uint8_t pf_num  = (reg & GL_MDET_RX_PF_NUM_M) >>
		    GL_MDET_RX_PF_NUM_S;
		uint16_t vf_num = (reg & GL_MDET_RX_VF_NUM_M) >>
		    GL_MDET_RX_VF_NUM_S;
		uint8_t event   = (reg & GL_MDET_RX_MAL_TYPE_M) >>
		    GL_MDET_RX_MAL_TYPE_S;
		uint16_t queue  = (reg & GL_MDET_RX_QNUM_M) >>
		    GL_MDET_RX_QNUM_S;

		printf("%s: Malicious Driver Detection Rx event '%s' "
		    "on Rx queue %u PF# %u VF# %u\n", sc->sc_dev.dv_xname,
		    ice_mdd_rx_str(event), queue, pf_num, vf_num);

		/* Only clear this event if it matches this PF, that way other
		 * PFs can read the event and determine VF and queue number.
		 */
		if (pf_num == hw->pf_id)
			ICE_WRITE(hw, GL_MDET_RX, 0xffffffff);

		mdd_detected = true;
	}

	/* Now, confirm that this event actually affects this PF, by checking
	 * the PF registers.
	 */
	if (mdd_detected) {
		reg = ICE_READ(hw, PF_MDET_TX_TCLAN);
		if (reg & PF_MDET_TX_TCLAN_VALID_M) {
			ICE_WRITE(hw, PF_MDET_TX_TCLAN, 0xffff);
#if 0
			sc->soft_stats.tx_mdd_count++;
#endif
			request_reinit = true;
		}

		reg = ICE_READ(hw, PF_MDET_TX_PQM);
		if (reg & PF_MDET_TX_PQM_VALID_M) {
			ICE_WRITE(hw, PF_MDET_TX_PQM, 0xffff);
#if 0
			sc->soft_stats.tx_mdd_count++;
#endif
			request_reinit = true;
		}

		reg = ICE_READ(hw, PF_MDET_RX);
		if (reg & PF_MDET_RX_VALID_M) {
			ICE_WRITE(hw, PF_MDET_RX, 0xffff);
#if 0
			sc->soft_stats.rx_mdd_count++;
#endif
			request_reinit = true;
		}
	}

	/* TODO: Implement logic to detect and handle events caused by VFs. */
	/* request that the upper stack re-initialize the Tx/Rx queues */
	if (request_reinit)
		ice_request_stack_reinit(sc);
	ice_flush(hw);
}

/**
 * ice_check_ctrlq_errors - Check for and report controlq errors
 * @sc: device private structure
 * @qname: name of the controlq
 * @cq: the controlq to check
 *
 * Check and report controlq errors. Currently all we do is report them to the
 * kernel message log, but we might want to improve this in the future, such
 * as to keep track of statistics.
 */
void
ice_check_ctrlq_errors(struct ice_softc *sc, const char *qname,
		       struct ice_ctl_q_info *cq)
{
	struct ice_hw *hw = &sc->hw;
	uint32_t val;

	/* Check for error indications. Note that all the controlqs use the
	 * same register layout, so we use the PF_FW_AxQLEN defines only.
	 */
	val = ICE_READ(hw, cq->rq.len);
	if (val & (PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M |
		   PF_FW_ARQLEN_ARQCRIT_M)) {
		if (val & PF_FW_ARQLEN_ARQVFE_M)
			printf("%s: %s Receive Queue VF Error detected\n",
			    sc->sc_dev.dv_xname, qname);
		if (val & PF_FW_ARQLEN_ARQOVFL_M)
			printf("%s: %s Receive Queue Overflow Error detected\n",
			    sc->sc_dev.dv_xname, qname);
		if (val & PF_FW_ARQLEN_ARQCRIT_M)
			printf("%s: %s Receive Queue Critical Error detected\n",
			    sc->sc_dev.dv_xname, qname);
		val &= ~(PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M |
			 PF_FW_ARQLEN_ARQCRIT_M);
		ICE_WRITE(hw, cq->rq.len, val);
	}

	val = ICE_READ(hw, cq->sq.len);
	if (val & (PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M |
		   PF_FW_ATQLEN_ATQCRIT_M)) {
		if (val & PF_FW_ATQLEN_ATQVFE_M)
			printf("%s: %s Send Queue VF Error detected\n",
			    sc->sc_dev.dv_xname, qname);
		if (val & PF_FW_ATQLEN_ATQOVFL_M)
			printf("%s: %s Send Queue Overflow Error detected\n",
			    sc->sc_dev.dv_xname, qname);
		if (val & PF_FW_ATQLEN_ATQCRIT_M)
			printf("%s: %s Send Queue Critical Error detected\n",
			    sc->sc_dev.dv_xname, qname);
		val &= ~(PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M |
			 PF_FW_ATQLEN_ATQCRIT_M);
		ICE_WRITE(hw, cq->sq.len, val);
	}
}

/**
 * ice_process_link_event - Process a link event indication from firmware
 * @sc: device softc structure
 * @e: the received event data
 *
 * Gets the current link status from hardware, and may print a message if an
 * unqualified is detected.
 */
void
ice_process_link_event(struct ice_softc *sc, struct ice_rq_event_info *e)
{
	struct ice_port_info *pi = sc->hw.port_info;
	struct ice_hw *hw = &sc->hw;
	enum ice_status status;

	/* Sanity check that the data length isn't too small */
	KASSERT(le16toh(e->desc.datalen) >= ICE_GET_LINK_STATUS_DATALEN_V1);

	/*
	 * Even though the adapter gets link status information inside the
	 * event, it needs to send a Get Link Status AQ command in order
	 * to re-enable link events.
	 */
	pi->phy.get_link_info = true;
	ice_get_link_status(pi, &sc->link_up);

	if (pi->phy.link_info.topo_media_conflict &
	   (ICE_AQ_LINK_TOPO_CONFLICT | ICE_AQ_LINK_MEDIA_CONFLICT |
	    ICE_AQ_LINK_TOPO_CORRUPT))
		printf("%s: Possible mis-configuration of the Ethernet port "
		    "detected: topology conflict, or link media conflict, "
		    "or link topology corrupt\n", sc->sc_dev.dv_xname);

	if ((pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) &&
	    !(pi->phy.link_info.link_info & ICE_AQ_LINK_UP)) {
		if (!(pi->phy.link_info.an_info & ICE_AQ_QUALIFIED_MODULE))
			printf("%s: Link is disabled on this device because "
			    "an unsupported module type was detected!",
			    sc->sc_dev.dv_xname);

		if (pi->phy.link_info.link_cfg_err &
		    ICE_AQ_LINK_MODULE_POWER_UNSUPPORTED)
			printf("%s: The module's power requirements exceed "
			    "the device's power supply. Cannot start link.\n",
			    sc->sc_dev.dv_xname);
		if (pi->phy.link_info.link_cfg_err &
		    ICE_AQ_LINK_INVAL_MAX_POWER_LIMIT)
			printf("%s: The installed module is incompatible with "
			    "the device's NVM image. Cannot start link.\n",
			    sc->sc_dev.dv_xname);
	}

	if (!(pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE)) {
		if (!ice_testandset_state(&sc->state, ICE_STATE_NO_MEDIA)) {
			status = ice_aq_set_link_restart_an(pi, false, NULL);
			if (status != ICE_SUCCESS &&
			    hw->adminq.sq_last_status != ICE_AQ_RC_EMODE)
				DPRINTF("%s: ice_aq_set_link_restart_an: "
				    "status %s, aq_err %s\n", __func__,
				    ice_status_str(status),
				    ice_aq_str(hw->adminq.sq_last_status));
		}
	}
	/* ICE_STATE_NO_MEDIA is cleared when polling task detects media */

	/* Indicate that link status must be reported again */
	ice_clear_state(&sc->state, ICE_STATE_LINK_STATUS_REPORTED);

	/* OS link info is updated elsewhere */
}

/**
 * ice_info_fwlog - Format and print an array of values to the console
 * @hw: private hardware structure
 * @rowsize: preferred number of rows to use
 * @groupsize: preferred size in bytes to print each chunk
 * @buf: the array buffer to print
 * @len: size of the array buffer
 *
 * Format the given array as a series of uint8_t values with hexadecimal
 * notation and log the contents to the console log.  This variation is
 * specific to firmware logging.
 *
 * TODO: Currently only supports a group size of 1, due to the way hexdump is
 * implemented.
 */
void
ice_info_fwlog(struct ice_hw *hw, uint32_t rowsize, uint32_t __unused groupsize,
	       uint8_t *buf, size_t len)
{
	struct ice_softc *sc = hw->hw_sc;

	if (!ice_fwlog_supported(hw))
		return;

	/* Format the device header to a string */
	printf("%s: FWLOG: ", sc->sc_dev.dv_xname);

	ice_hexdump(buf, len);
}

/**
 * ice_fwlog_event_dump - Dump the event received over the Admin Receive Queue
 * @hw: pointer to the HW structure
 * @desc: Admin Receive Queue descriptor
 * @buf: buffer that contains the FW log event data
 *
 * If the driver receives the ice_aqc_opc_fw_logs_event on the Admin Receive
 * Queue, then it should call this function to dump the FW log data.
 */
void
ice_fwlog_event_dump(struct ice_hw *hw, struct ice_aq_desc *desc, void *buf)
{
	if (!ice_fwlog_supported(hw))
		return;

	ice_info_fwlog(hw, 32, 1, (uint8_t *)buf, le16toh(desc->datalen));
}

/**
 * ice_handle_fw_log_event - Handle a firmware logging event from the AdminQ
 * @sc: pointer to private softc structure
 * @desc: the AdminQ descriptor for this firmware event
 * @buf: pointer to the buffer accompanying the AQ message
 */
void
ice_handle_fw_log_event(struct ice_softc *sc, struct ice_aq_desc *desc,
			void *buf)
{
#if 0
	/* Trigger a DTrace probe event for this firmware message */
	SDT_PROBE2(ice_fwlog, , , message, (const u8 *)buf, desc->datalen);
#endif
	/* Possibly dump the firmware message to the console, if enabled */
	ice_fwlog_event_dump(&sc->hw, desc, buf);
}

/**
 * ice_debug_print_mib_change_event - helper function to log LLDP MIB change events
 * @sc: the device private softc
 * @event: event received on a control queue
 *
 * Prints out the type and contents of an LLDP MIB change event in a DCB debug message.
 */
void
ice_debug_print_mib_change_event(struct ice_softc *sc,
    struct ice_rq_event_info *event)
{
#ifdef ICE_DEBUG
	struct ice_aqc_lldp_get_mib *params =
	    (struct ice_aqc_lldp_get_mib *)&event->desc.params.lldp_get_mib;
	uint8_t mib_type, bridge_type, tx_status;

	static const char* mib_type_strings[] = {
	    "Local MIB",
	    "Remote MIB",
	    "Reserved",
	    "Reserved"
	};
	static const char* bridge_type_strings[] = {
	    "Nearest Bridge",
	    "Non-TPMR Bridge",
	    "Reserved",
	    "Reserved"
	};
	static const char* tx_status_strings[] = {
	    "Port's TX active",
	    "Port's TX suspended and drained",
	    "Reserved",
	    "Port's TX suspended and drained; blocked TC pipe flushed"
	};

	mib_type = (params->type & ICE_AQ_LLDP_MIB_TYPE_M) >>
	    ICE_AQ_LLDP_MIB_TYPE_S;
	bridge_type = (params->type & ICE_AQ_LLDP_BRID_TYPE_M) >>
	    ICE_AQ_LLDP_BRID_TYPE_S;
	tx_status = (params->type & ICE_AQ_LLDP_TX_M) >>
	    ICE_AQ_LLDP_TX_S;

	DNPRINTF(ICE_DBG_DCB, "%s: LLDP MIB Change Event (%s, %s, %s)\n",
	    sc->sc_dev.dv_xname,
	    mib_type_strings[mib_type], bridge_type_strings[bridge_type],
	    tx_status_strings[tx_status]);

	/* Nothing else to report */
	if (!event->msg_buf)
		return;

	DNPRINTF(ICE_DBG_DCB, "- %s contents:\n", mib_type_strings[mib_type]);
	ice_debug_array(&sc->hw, ICE_DBG_DCB, 16, 1, event->msg_buf,
			event->msg_len);
#endif
}

/**
 * ice_aq_get_lldp_mib
 * @hw: pointer to the HW struct
 * @bridge_type: type of bridge requested
 * @mib_type: Local, Remote or both Local and Remote MIBs
 * @buf: pointer to the caller-supplied buffer to store the MIB block
 * @buf_size: size of the buffer (in bytes)
 * @local_len: length of the returned Local LLDP MIB
 * @remote_len: length of the returned Remote LLDP MIB
 * @cd: pointer to command details structure or NULL
 *
 * Requests the complete LLDP MIB (entire packet). (0x0A00)
 */
enum ice_status
ice_aq_get_lldp_mib(struct ice_hw *hw, uint8_t bridge_type, uint8_t mib_type,
    void *buf, uint16_t buf_size, uint16_t *local_len, uint16_t *remote_len,
    struct ice_sq_cd *cd)
{
	struct ice_aqc_lldp_get_mib *cmd;
	struct ice_aq_desc desc;
	enum ice_status status;

	cmd = &desc.params.lldp_get_mib;

	if (buf_size == 0 || !buf)
		return ICE_ERR_PARAM;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_lldp_get_mib);

	cmd->type = mib_type & ICE_AQ_LLDP_MIB_TYPE_M;
	cmd->type |= (bridge_type << ICE_AQ_LLDP_BRID_TYPE_S) &
		ICE_AQ_LLDP_BRID_TYPE_M;

	desc.datalen = htole16(buf_size);

	status = ice_aq_send_cmd(hw, &desc, buf, buf_size, cd);
	if (!status) {
		if (local_len)
			*local_len = le16toh(cmd->local_len);
		if (remote_len)
			*remote_len = le16toh(cmd->remote_len);
	}

	return status;
}

/**
 * ice_parse_ieee_ets_common_tlv
 * @buf: Data buffer to be parsed for ETS CFG/REC data
 * @ets_cfg: Container to store parsed data
 *
 * Parses the common data of IEEE 802.1Qaz ETS CFG/REC TLV
 */
void
ice_parse_ieee_ets_common_tlv(uint8_t *buf, struct ice_dcb_ets_cfg *ets_cfg)
{
	uint8_t offset = 0;
	int i;

	/* Priority Assignment Table (4 octets)
	 * Octets:|    1    |    2    |    3    |    4    |
	 *        -----------------------------------------
	 *        |pri0|pri1|pri2|pri3|pri4|pri5|pri6|pri7|
	 *        -----------------------------------------
	 *   Bits:|7  4|3  0|7  4|3  0|7  4|3  0|7  4|3  0|
	 *        -----------------------------------------
	 */
	for (i = 0; i < 4; i++) {
		ets_cfg->prio_table[i * 2] =
			((buf[offset] & ICE_IEEE_ETS_PRIO_1_M) >>
			 ICE_IEEE_ETS_PRIO_1_S);
		ets_cfg->prio_table[i * 2 + 1] =
			((buf[offset] & ICE_IEEE_ETS_PRIO_0_M) >>
			 ICE_IEEE_ETS_PRIO_0_S);
		offset++;
	}

	/* TC Bandwidth Table (8 octets)
	 * Octets:| 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 |
	 *        ---------------------------------
	 *        |tc0|tc1|tc2|tc3|tc4|tc5|tc6|tc7|
	 *        ---------------------------------
	 *
	 * TSA Assignment Table (8 octets)
	 * Octets:| 9 | 10| 11| 12| 13| 14| 15| 16|
	 *        ---------------------------------
	 *        |tc0|tc1|tc2|tc3|tc4|tc5|tc6|tc7|
	 *        ---------------------------------
	 */
	ice_for_each_traffic_class(i) {
		ets_cfg->tcbwtable[i] = buf[offset];
		ets_cfg->tsatable[i] = buf[ICE_MAX_TRAFFIC_CLASS + offset++];
	}
}

/**
 * ice_parse_ieee_etscfg_tlv
 * @tlv: IEEE 802.1Qaz ETS CFG TLV
 * @dcbcfg: Local store to update ETS CFG data
 *
 * Parses IEEE 802.1Qaz ETS CFG TLV
 */
void
ice_parse_ieee_etscfg_tlv(struct ice_lldp_org_tlv *tlv,
			  struct ice_dcbx_cfg *dcbcfg)
{
	struct ice_dcb_ets_cfg *etscfg;
	uint8_t *buf = tlv->tlvinfo;

	/* First Octet post subtype
	 * --------------------------
	 * |will-|CBS  | Re-  | Max |
	 * |ing  |     |served| TCs |
	 * --------------------------
	 * |1bit | 1bit|3 bits|3bits|
	 */
	etscfg = &dcbcfg->etscfg;
	etscfg->willing = ((buf[0] & ICE_IEEE_ETS_WILLING_M) >>
			   ICE_IEEE_ETS_WILLING_S);
	etscfg->cbs = ((buf[0] & ICE_IEEE_ETS_CBS_M) >> ICE_IEEE_ETS_CBS_S);
	etscfg->maxtcs = ((buf[0] & ICE_IEEE_ETS_MAXTC_M) >>
			  ICE_IEEE_ETS_MAXTC_S);

	/* Begin parsing at Priority Assignment Table (offset 1 in buf) */
	ice_parse_ieee_ets_common_tlv(&buf[1], etscfg);
}

/**
 * ice_parse_ieee_etsrec_tlv
 * @tlv: IEEE 802.1Qaz ETS REC TLV
 * @dcbcfg: Local store to update ETS REC data
 *
 * Parses IEEE 802.1Qaz ETS REC TLV
 */
void
ice_parse_ieee_etsrec_tlv(struct ice_lldp_org_tlv *tlv,
			  struct ice_dcbx_cfg *dcbcfg)
{
	uint8_t *buf = tlv->tlvinfo;

	/* Begin parsing at Priority Assignment Table (offset 1 in buf) */
	ice_parse_ieee_ets_common_tlv(&buf[1], &dcbcfg->etsrec);
}

/**
 * ice_parse_ieee_pfccfg_tlv
 * @tlv: IEEE 802.1Qaz PFC CFG TLV
 * @dcbcfg: Local store to update PFC CFG data
 *
 * Parses IEEE 802.1Qaz PFC CFG TLV
 */
void
ice_parse_ieee_pfccfg_tlv(struct ice_lldp_org_tlv *tlv,
			  struct ice_dcbx_cfg *dcbcfg)
{
	uint8_t *buf = tlv->tlvinfo;

	/* ----------------------------------------
	 * |will-|MBC  | Re-  | PFC |  PFC Enable  |
	 * |ing  |     |served| cap |              |
	 * -----------------------------------------
	 * |1bit | 1bit|2 bits|4bits| 1 octet      |
	 */
	dcbcfg->pfc.willing = ((buf[0] & ICE_IEEE_PFC_WILLING_M) >>
			       ICE_IEEE_PFC_WILLING_S);
	dcbcfg->pfc.mbc = ((buf[0] & ICE_IEEE_PFC_MBC_M) >> ICE_IEEE_PFC_MBC_S);
	dcbcfg->pfc.pfccap = ((buf[0] & ICE_IEEE_PFC_CAP_M) >>
			      ICE_IEEE_PFC_CAP_S);
	dcbcfg->pfc.pfcena = buf[1];
}

/**
 * ice_parse_ieee_app_tlv
 * @tlv: IEEE 802.1Qaz APP TLV
 * @dcbcfg: Local store to update APP PRIO data
 *
 * Parses IEEE 802.1Qaz APP PRIO TLV
 */
void
ice_parse_ieee_app_tlv(struct ice_lldp_org_tlv *tlv,
		       struct ice_dcbx_cfg *dcbcfg)
{
	uint16_t offset = 0;
	uint16_t typelen;
	int i = 0;
	uint16_t len;
	uint8_t *buf;

	typelen = ntohs(tlv->typelen);
	len = ((typelen & ICE_LLDP_TLV_LEN_M) >> ICE_LLDP_TLV_LEN_S);
	buf = tlv->tlvinfo;

	/* Removing sizeof(ouisubtype) and reserved byte from len.
	 * Remaining len div 3 is number of APP TLVs.
	 */
	len -= (sizeof(tlv->ouisubtype) + 1);

	/* Move offset to App Priority Table */
	offset++;

	/* Application Priority Table (3 octets)
	 * Octets:|         1          |    2    |    3    |
	 *        -----------------------------------------
	 *        |Priority|Rsrvd| Sel |    Protocol ID    |
	 *        -----------------------------------------
	 *   Bits:|23    21|20 19|18 16|15                0|
	 *        -----------------------------------------
	 */
	while (offset < len) {
		dcbcfg->app[i].priority = ((buf[offset] &
					    ICE_IEEE_APP_PRIO_M) >>
					   ICE_IEEE_APP_PRIO_S);
		dcbcfg->app[i].selector = ((buf[offset] &
					    ICE_IEEE_APP_SEL_M) >>
					   ICE_IEEE_APP_SEL_S);
		dcbcfg->app[i].prot_id = (buf[offset + 1] << 0x8) |
			buf[offset + 2];
		/* Move to next app */
		offset += 3;
		i++;
		if (i >= ICE_DCBX_MAX_APPS)
			break;
	}

	dcbcfg->numapps = i;
}

/**
 * ice_parse_ieee_tlv
 * @tlv: IEEE 802.1Qaz TLV
 * @dcbcfg: Local store to update ETS REC data
 *
 * Get the TLV subtype and send it to parsing function
 * based on the subtype value
 */
void
ice_parse_ieee_tlv(struct ice_lldp_org_tlv *tlv, struct ice_dcbx_cfg *dcbcfg)
{
	uint32_t ouisubtype;
	uint8_t subtype;

	ouisubtype = ntohl(tlv->ouisubtype);
	subtype = (uint8_t)((ouisubtype & ICE_LLDP_TLV_SUBTYPE_M) >>
		       ICE_LLDP_TLV_SUBTYPE_S);
	switch (subtype) {
	case ICE_IEEE_SUBTYPE_ETS_CFG:
		ice_parse_ieee_etscfg_tlv(tlv, dcbcfg);
		break;
	case ICE_IEEE_SUBTYPE_ETS_REC:
		ice_parse_ieee_etsrec_tlv(tlv, dcbcfg);
		break;
	case ICE_IEEE_SUBTYPE_PFC_CFG:
		ice_parse_ieee_pfccfg_tlv(tlv, dcbcfg);
		break;
	case ICE_IEEE_SUBTYPE_APP_PRI:
		ice_parse_ieee_app_tlv(tlv, dcbcfg);
		break;
	default:
		break;
	}
}

/**
 * ice_parse_cee_pgcfg_tlv
 * @tlv: CEE DCBX PG CFG TLV
 * @dcbcfg: Local store to update ETS CFG data
 *
 * Parses CEE DCBX PG CFG TLV
 */
void
ice_parse_cee_pgcfg_tlv(struct ice_cee_feat_tlv *tlv,
			struct ice_dcbx_cfg *dcbcfg)
{
	struct ice_dcb_ets_cfg *etscfg;
	uint8_t *buf = tlv->tlvinfo;
	uint16_t offset = 0;
	int i;

	etscfg = &dcbcfg->etscfg;

	if (tlv->en_will_err & ICE_CEE_FEAT_TLV_WILLING_M)
		etscfg->willing = 1;

	etscfg->cbs = 0;
	/* Priority Group Table (4 octets)
	 * Octets:|    1    |    2    |    3    |    4    |
	 *        -----------------------------------------
	 *        |pri0|pri1|pri2|pri3|pri4|pri5|pri6|pri7|
	 *        -----------------------------------------
	 *   Bits:|7  4|3  0|7  4|3  0|7  4|3  0|7  4|3  0|
	 *        -----------------------------------------
	 */
	for (i = 0; i < 4; i++) {
		etscfg->prio_table[i * 2] =
			((buf[offset] & ICE_CEE_PGID_PRIO_1_M) >>
			 ICE_CEE_PGID_PRIO_1_S);
		etscfg->prio_table[i * 2 + 1] =
			((buf[offset] & ICE_CEE_PGID_PRIO_0_M) >>
			 ICE_CEE_PGID_PRIO_0_S);
		offset++;
	}

	/* PG Percentage Table (8 octets)
	 * Octets:| 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 |
	 *        ---------------------------------
	 *        |pg0|pg1|pg2|pg3|pg4|pg5|pg6|pg7|
	 *        ---------------------------------
	 */
	ice_for_each_traffic_class(i) {
		etscfg->tcbwtable[i] = buf[offset++];

		if (etscfg->prio_table[i] == ICE_CEE_PGID_STRICT)
			dcbcfg->etscfg.tsatable[i] = ICE_IEEE_TSA_STRICT;
		else
			dcbcfg->etscfg.tsatable[i] = ICE_IEEE_TSA_ETS;
	}

	/* Number of TCs supported (1 octet) */
	etscfg->maxtcs = buf[offset];
}

/**
 * ice_parse_cee_pfccfg_tlv
 * @tlv: CEE DCBX PFC CFG TLV
 * @dcbcfg: Local store to update PFC CFG data
 *
 * Parses CEE DCBX PFC CFG TLV
 */
void
ice_parse_cee_pfccfg_tlv(struct ice_cee_feat_tlv *tlv,
			 struct ice_dcbx_cfg *dcbcfg)
{
	uint8_t *buf = tlv->tlvinfo;

	if (tlv->en_will_err & ICE_CEE_FEAT_TLV_WILLING_M)
		dcbcfg->pfc.willing = 1;

	/* ------------------------
	 * | PFC Enable | PFC TCs |
	 * ------------------------
	 * | 1 octet    | 1 octet |
	 */
	dcbcfg->pfc.pfcena = buf[0];
	dcbcfg->pfc.pfccap = buf[1];
}

/**
 * ice_parse_cee_app_tlv
 * @tlv: CEE DCBX APP TLV
 * @dcbcfg: Local store to update APP PRIO data
 *
 * Parses CEE DCBX APP PRIO TLV
 */
void
ice_parse_cee_app_tlv(struct ice_cee_feat_tlv *tlv, struct ice_dcbx_cfg *dcbcfg)
{
	uint16_t len, typelen, offset = 0;
	struct ice_cee_app_prio *app;
	uint8_t i;

	typelen = NTOHS(tlv->hdr.typelen);
	len = ((typelen & ICE_LLDP_TLV_LEN_M) >> ICE_LLDP_TLV_LEN_S);

	dcbcfg->numapps = len / sizeof(*app);
	if (!dcbcfg->numapps)
		return;
	if (dcbcfg->numapps > ICE_DCBX_MAX_APPS)
		dcbcfg->numapps = ICE_DCBX_MAX_APPS;

	for (i = 0; i < dcbcfg->numapps; i++) {
		uint8_t up, selector;

		app = (struct ice_cee_app_prio *)(tlv->tlvinfo + offset);
		for (up = 0; up < ICE_MAX_USER_PRIORITY; up++)
			if (app->prio_map & BIT(up))
				break;

		dcbcfg->app[i].priority = up;

		/* Get Selector from lower 2 bits, and convert to IEEE */
		selector = (app->upper_oui_sel & ICE_CEE_APP_SELECTOR_M);
		switch (selector) {
		case ICE_CEE_APP_SEL_ETHTYPE:
			dcbcfg->app[i].selector = ICE_APP_SEL_ETHTYPE;
			break;
		case ICE_CEE_APP_SEL_TCPIP:
			dcbcfg->app[i].selector = ICE_APP_SEL_TCPIP;
			break;
		default:
			/* Keep selector as it is for unknown types */
			dcbcfg->app[i].selector = selector;
		}

		dcbcfg->app[i].prot_id = NTOHS(app->protocol);
		/* Move to next app */
		offset += sizeof(*app);
	}
}

/**
 * ice_parse_cee_tlv
 * @tlv: CEE DCBX TLV
 * @dcbcfg: Local store to update DCBX config data
 *
 * Get the TLV subtype and send it to parsing function
 * based on the subtype value
 */
void
ice_parse_cee_tlv(struct ice_lldp_org_tlv *tlv, struct ice_dcbx_cfg *dcbcfg)
{
	struct ice_cee_feat_tlv *sub_tlv;
	uint8_t subtype, feat_tlv_count = 0;
	uint16_t len, tlvlen, typelen;
	uint32_t ouisubtype;

	ouisubtype = ntohl(tlv->ouisubtype);
	subtype = (uint8_t)((ouisubtype & ICE_LLDP_TLV_SUBTYPE_M) >>
		       ICE_LLDP_TLV_SUBTYPE_S);
	/* Return if not CEE DCBX */
	if (subtype != ICE_CEE_DCBX_TYPE)
		return;

	typelen = ntohs(tlv->typelen);
	tlvlen = ((typelen & ICE_LLDP_TLV_LEN_M) >> ICE_LLDP_TLV_LEN_S);
	len = sizeof(tlv->typelen) + sizeof(ouisubtype) +
		sizeof(struct ice_cee_ctrl_tlv);
	/* Return if no CEE DCBX Feature TLVs */
	if (tlvlen <= len)
		return;

	sub_tlv = (struct ice_cee_feat_tlv *)((char *)tlv + len);
	while (feat_tlv_count < ICE_CEE_MAX_FEAT_TYPE) {
		uint16_t sublen;

		typelen = ntohs(sub_tlv->hdr.typelen);
		sublen = ((typelen & ICE_LLDP_TLV_LEN_M) >> ICE_LLDP_TLV_LEN_S);
		subtype = (uint8_t)((typelen & ICE_LLDP_TLV_TYPE_M) >>
			       ICE_LLDP_TLV_TYPE_S);
		switch (subtype) {
		case ICE_CEE_SUBTYPE_PG_CFG:
			ice_parse_cee_pgcfg_tlv(sub_tlv, dcbcfg);
			break;
		case ICE_CEE_SUBTYPE_PFC_CFG:
			ice_parse_cee_pfccfg_tlv(sub_tlv, dcbcfg);
			break;
		case ICE_CEE_SUBTYPE_APP_PRI:
			ice_parse_cee_app_tlv(sub_tlv, dcbcfg);
			break;
		default:
			return;	/* Invalid Sub-type return */
		}
		feat_tlv_count++;
		/* Move to next sub TLV */
		sub_tlv = (struct ice_cee_feat_tlv *)
			  ((char *)sub_tlv + sizeof(sub_tlv->hdr.typelen) +
			   sublen);
	}
}
/**
 * ice_parse_org_tlv
 * @tlv: Organization specific TLV
 * @dcbcfg: Local store to update ETS REC data
 *
 * Currently only IEEE 802.1Qaz TLV is supported, all others
 * will be returned
 */
void
ice_parse_org_tlv(struct ice_lldp_org_tlv *tlv, struct ice_dcbx_cfg *dcbcfg)
{
	uint32_t ouisubtype;
	uint32_t oui;

	ouisubtype = ntohl(tlv->ouisubtype);
	oui = ((ouisubtype & ICE_LLDP_TLV_OUI_M) >> ICE_LLDP_TLV_OUI_S);
	switch (oui) {
	case ICE_IEEE_8021QAZ_OUI:
		ice_parse_ieee_tlv(tlv, dcbcfg);
		break;
	case ICE_CEE_DCBX_OUI:
		ice_parse_cee_tlv(tlv, dcbcfg);
		break;
	default:
		break;
	}
}
/**
 * ice_lldp_to_dcb_cfg
 * @lldpmib: LLDPDU to be parsed
 * @dcbcfg: store for LLDPDU data
 *
 * Parse DCB configuration from the LLDPDU
 */
enum ice_status
ice_lldp_to_dcb_cfg(uint8_t *lldpmib, struct ice_dcbx_cfg *dcbcfg)
{
	struct ice_lldp_org_tlv *tlv;
	enum ice_status ret = ICE_SUCCESS;
	uint16_t offset = 0;
	uint16_t typelen;
	uint16_t type;
	uint16_t len;

	if (!lldpmib || !dcbcfg)
		return ICE_ERR_PARAM;

	/* set to the start of LLDPDU */
	lldpmib += ETHER_HDR_LEN;
	tlv = (struct ice_lldp_org_tlv *)lldpmib;
	while (1) {
		typelen = ntohs(tlv->typelen);
		type = ((typelen & ICE_LLDP_TLV_TYPE_M) >> ICE_LLDP_TLV_TYPE_S);
		len = ((typelen & ICE_LLDP_TLV_LEN_M) >> ICE_LLDP_TLV_LEN_S);
		offset += sizeof(typelen) + len;

		/* END TLV or beyond LLDPDU size */
		if (type == ICE_TLV_TYPE_END || offset > ICE_LLDPDU_SIZE)
			break;

		switch (type) {
		case ICE_TLV_TYPE_ORG:
			ice_parse_org_tlv(tlv, dcbcfg);
			break;
		default:
			break;
		}

		/* Move to next TLV */
		tlv = (struct ice_lldp_org_tlv *)
		      ((char *)tlv + sizeof(tlv->typelen) + len);
	}

	return ret;
}
/**
 * ice_aq_get_dcb_cfg
 * @hw: pointer to the HW struct
 * @mib_type: MIB type for the query
 * @bridgetype: bridge type for the query (remote)
 * @dcbcfg: store for LLDPDU data
 *
 * Query DCB configuration from the firmware
 */
enum ice_status
ice_aq_get_dcb_cfg(struct ice_hw *hw, uint8_t mib_type, uint8_t bridgetype,
		   struct ice_dcbx_cfg *dcbcfg)
{
	enum ice_status ret;
	uint8_t *lldpmib;

	/* Allocate the LLDPDU */
	lldpmib = (uint8_t *)ice_malloc(hw, ICE_LLDPDU_SIZE);
	if (!lldpmib)
		return ICE_ERR_NO_MEMORY;

	ret = ice_aq_get_lldp_mib(hw, bridgetype, mib_type, (void *)lldpmib,
				  ICE_LLDPDU_SIZE, NULL, NULL, NULL);

	if (ret == ICE_SUCCESS)
		/* Parse LLDP MIB to get DCB configuration */
		ret = ice_lldp_to_dcb_cfg(lldpmib, dcbcfg);

	ice_free(hw, lldpmib);

	return ret;
}

/**
 * ice_cee_to_dcb_cfg
 * @cee_cfg: pointer to CEE configuration struct
 * @pi: port information structure
 *
 * Convert CEE configuration from firmware to DCB configuration
 */
void
ice_cee_to_dcb_cfg(struct ice_aqc_get_cee_dcb_cfg_resp *cee_cfg,
		   struct ice_port_info *pi)
{
	uint32_t status, tlv_status = le32toh(cee_cfg->tlv_status);
	uint32_t ice_aqc_cee_status_mask, ice_aqc_cee_status_shift;
	uint8_t i, j, err, sync, oper, app_index, ice_app_sel_type;
	uint16_t app_prio = le16toh(cee_cfg->oper_app_prio);
	uint16_t ice_aqc_cee_app_mask, ice_aqc_cee_app_shift;
	struct ice_dcbx_cfg *cmp_dcbcfg, *dcbcfg;
	uint16_t ice_app_prot_id_type;

	dcbcfg = &pi->qos_cfg.local_dcbx_cfg;
	dcbcfg->dcbx_mode = ICE_DCBX_MODE_CEE;
	dcbcfg->tlv_status = tlv_status;

	/* CEE PG data */
	dcbcfg->etscfg.maxtcs = cee_cfg->oper_num_tc;

	/* Note that the FW creates the oper_prio_tc nibbles reversed
	 * from those in the CEE Priority Group sub-TLV.
	 */
	for (i = 0; i < ICE_MAX_TRAFFIC_CLASS / 2; i++) {
		dcbcfg->etscfg.prio_table[i * 2] =
			((cee_cfg->oper_prio_tc[i] & ICE_CEE_PGID_PRIO_0_M) >>
			 ICE_CEE_PGID_PRIO_0_S);
		dcbcfg->etscfg.prio_table[i * 2 + 1] =
			((cee_cfg->oper_prio_tc[i] & ICE_CEE_PGID_PRIO_1_M) >>
			 ICE_CEE_PGID_PRIO_1_S);
	}

	ice_for_each_traffic_class(i) {
		dcbcfg->etscfg.tcbwtable[i] = cee_cfg->oper_tc_bw[i];

		if (dcbcfg->etscfg.prio_table[i] == ICE_CEE_PGID_STRICT) {
			/* Map it to next empty TC */
			dcbcfg->etscfg.prio_table[i] = cee_cfg->oper_num_tc - 1;
			dcbcfg->etscfg.tsatable[i] = ICE_IEEE_TSA_STRICT;
		} else {
			dcbcfg->etscfg.tsatable[i] = ICE_IEEE_TSA_ETS;
		}
	}

	/* CEE PFC data */
	dcbcfg->pfc.pfcena = cee_cfg->oper_pfc_en;
	dcbcfg->pfc.pfccap = ICE_MAX_TRAFFIC_CLASS;

	/* CEE APP TLV data */
	if (dcbcfg->app_mode == ICE_DCBX_APPS_NON_WILLING)
		cmp_dcbcfg = &pi->qos_cfg.desired_dcbx_cfg;
	else
		cmp_dcbcfg = &pi->qos_cfg.remote_dcbx_cfg;

	app_index = 0;
	for (i = 0; i < 3; i++) {
		if (i == 0) {
			/* FCoE APP */
			ice_aqc_cee_status_mask = ICE_AQC_CEE_FCOE_STATUS_M;
			ice_aqc_cee_status_shift = ICE_AQC_CEE_FCOE_STATUS_S;
			ice_aqc_cee_app_mask = ICE_AQC_CEE_APP_FCOE_M;
			ice_aqc_cee_app_shift = ICE_AQC_CEE_APP_FCOE_S;
			ice_app_sel_type = ICE_APP_SEL_ETHTYPE;
			ice_app_prot_id_type = ICE_APP_PROT_ID_FCOE;
		} else if (i == 1) {
			/* iSCSI APP */
			ice_aqc_cee_status_mask = ICE_AQC_CEE_ISCSI_STATUS_M;
			ice_aqc_cee_status_shift = ICE_AQC_CEE_ISCSI_STATUS_S;
			ice_aqc_cee_app_mask = ICE_AQC_CEE_APP_ISCSI_M;
			ice_aqc_cee_app_shift = ICE_AQC_CEE_APP_ISCSI_S;
			ice_app_sel_type = ICE_APP_SEL_TCPIP;
			ice_app_prot_id_type = ICE_APP_PROT_ID_ISCSI;

			for (j = 0; j < cmp_dcbcfg->numapps; j++) {
				uint16_t prot_id = cmp_dcbcfg->app[j].prot_id;
				uint8_t sel = cmp_dcbcfg->app[j].selector;

				if  (sel == ICE_APP_SEL_TCPIP &&
				     (prot_id == ICE_APP_PROT_ID_ISCSI ||
				      prot_id == ICE_APP_PROT_ID_ISCSI_860)) {
					ice_app_prot_id_type = prot_id;
					break;
				}
			}
		} else {
			/* FIP APP */
			ice_aqc_cee_status_mask = ICE_AQC_CEE_FIP_STATUS_M;
			ice_aqc_cee_status_shift = ICE_AQC_CEE_FIP_STATUS_S;
			ice_aqc_cee_app_mask = ICE_AQC_CEE_APP_FIP_M;
			ice_aqc_cee_app_shift = ICE_AQC_CEE_APP_FIP_S;
			ice_app_sel_type = ICE_APP_SEL_ETHTYPE;
			ice_app_prot_id_type = ICE_APP_PROT_ID_FIP;
		}

		status = (tlv_status & ice_aqc_cee_status_mask) >>
			 ice_aqc_cee_status_shift;
		err = (status & ICE_TLV_STATUS_ERR) ? 1 : 0;
		sync = (status & ICE_TLV_STATUS_SYNC) ? 1 : 0;
		oper = (status & ICE_TLV_STATUS_OPER) ? 1 : 0;
		/* Add FCoE/iSCSI/FIP APP if Error is False and
		 * Oper/Sync is True
		 */
		if (!err && sync && oper) {
			dcbcfg->app[app_index].priority =
				(uint8_t)((app_prio & ice_aqc_cee_app_mask) >>
				     ice_aqc_cee_app_shift);
			dcbcfg->app[app_index].selector = ice_app_sel_type;
			dcbcfg->app[app_index].prot_id = ice_app_prot_id_type;
			app_index++;
		}
	}

	dcbcfg->numapps = app_index;
}

/**
 * ice_get_dcb_cfg_from_mib_change
 * @pi: port information structure
 * @event: pointer to the admin queue receive event
 *
 * Set DCB configuration from received MIB Change event
 */
void
ice_get_dcb_cfg_from_mib_change(struct ice_port_info *pi,
    struct ice_rq_event_info *event)
{
	struct ice_dcbx_cfg *dcbx_cfg = &pi->qos_cfg.local_dcbx_cfg;
	struct ice_aqc_lldp_get_mib *mib;
	uint8_t change_type, dcbx_mode;

	mib = (struct ice_aqc_lldp_get_mib *)&event->desc.params.raw;

	change_type = mib->type & ICE_AQ_LLDP_MIB_TYPE_M;
	if (change_type == ICE_AQ_LLDP_MIB_REMOTE)
		dcbx_cfg = &pi->qos_cfg.remote_dcbx_cfg;

	dcbx_mode = ((mib->type & ICE_AQ_LLDP_DCBX_M) >>
		     ICE_AQ_LLDP_DCBX_S);

	switch (dcbx_mode) {
	case ICE_AQ_LLDP_DCBX_IEEE:
		dcbx_cfg->dcbx_mode = ICE_DCBX_MODE_IEEE;
		ice_lldp_to_dcb_cfg(event->msg_buf, dcbx_cfg);
		break;

	case ICE_AQ_LLDP_DCBX_CEE:
		pi->qos_cfg.desired_dcbx_cfg = pi->qos_cfg.local_dcbx_cfg;
		ice_cee_to_dcb_cfg((struct ice_aqc_get_cee_dcb_cfg_resp *)
				   event->msg_buf, pi);
		break;
	}
}

/**
 * ice_aq_get_cee_dcb_cfg
 * @hw: pointer to the HW struct
 * @buff: response buffer that stores CEE operational configuration
 * @cd: pointer to command details structure or NULL
 *
 * Get CEE DCBX mode operational configuration from firmware (0x0A07)
 */
enum ice_status
ice_aq_get_cee_dcb_cfg(struct ice_hw *hw,
		       struct ice_aqc_get_cee_dcb_cfg_resp *buff,
		       struct ice_sq_cd *cd)
{
	struct ice_aq_desc desc;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_get_cee_dcb_cfg);

	return ice_aq_send_cmd(hw, &desc, (void *)buff, sizeof(*buff), cd);
}

/**
 * ice_get_ieee_or_cee_dcb_cfg
 * @pi: port information structure
 * @dcbx_mode: mode of DCBX (IEEE or CEE)
 *
 * Get IEEE or CEE mode DCB configuration from the Firmware
 */
enum ice_status
ice_get_ieee_or_cee_dcb_cfg(struct ice_port_info *pi, uint8_t dcbx_mode)
{
	struct ice_dcbx_cfg *dcbx_cfg = NULL;
	enum ice_status ret;

	if (!pi)
		return ICE_ERR_PARAM;

	if (dcbx_mode == ICE_DCBX_MODE_IEEE)
		dcbx_cfg = &pi->qos_cfg.local_dcbx_cfg;
	else if (dcbx_mode == ICE_DCBX_MODE_CEE)
		dcbx_cfg = &pi->qos_cfg.desired_dcbx_cfg;

	/* Get Local DCB Config in case of ICE_DCBX_MODE_IEEE
	 * or get CEE DCB Desired Config in case of ICE_DCBX_MODE_CEE
	 */
	ret = ice_aq_get_dcb_cfg(pi->hw, ICE_AQ_LLDP_MIB_LOCAL,
				 ICE_AQ_LLDP_BRID_TYPE_NEAREST_BRID, dcbx_cfg);
	if (ret)
		goto out;

	/* Get Remote DCB Config */
	dcbx_cfg = &pi->qos_cfg.remote_dcbx_cfg;
	ret = ice_aq_get_dcb_cfg(pi->hw, ICE_AQ_LLDP_MIB_REMOTE,
				 ICE_AQ_LLDP_BRID_TYPE_NEAREST_BRID, dcbx_cfg);
	/* Don't treat ENOENT as an error for Remote MIBs */
	if (pi->hw->adminq.sq_last_status == ICE_AQ_RC_ENOENT)
		ret = ICE_SUCCESS;

out:
	return ret;
}

/**
 * ice_get_dcb_cfg
 * @pi: port information structure
 *
 * Get DCB configuration from the Firmware
 */
enum ice_status
ice_get_dcb_cfg(struct ice_port_info *pi)
{
	struct ice_aqc_get_cee_dcb_cfg_resp cee_cfg;
	struct ice_dcbx_cfg *dcbx_cfg;
	enum ice_status ret;

	if (!pi)
		return ICE_ERR_PARAM;

	ret = ice_aq_get_cee_dcb_cfg(pi->hw, &cee_cfg, NULL);
	if (ret == ICE_SUCCESS) {
		/* CEE mode */
		ret = ice_get_ieee_or_cee_dcb_cfg(pi, ICE_DCBX_MODE_CEE);
		ice_cee_to_dcb_cfg(&cee_cfg, pi);
	} else if (pi->hw->adminq.sq_last_status == ICE_AQ_RC_ENOENT) {
		/* CEE mode not enabled try querying IEEE data */
		dcbx_cfg = &pi->qos_cfg.local_dcbx_cfg;
		dcbx_cfg->dcbx_mode = ICE_DCBX_MODE_IEEE;
		ret = ice_get_ieee_or_cee_dcb_cfg(pi, ICE_DCBX_MODE_IEEE);
	}

	return ret;
}

/**
 * ice_dcb_needs_reconfig - Returns true if driver needs to reconfigure
 * @sc: the device private softc
 * @old_cfg: Old DCBX configuration to compare against
 * @new_cfg: New DCBX configuration to check
 *
 * @return true if something changed in new_cfg that requires the driver
 * to do some reconfiguration.
 */
bool
ice_dcb_needs_reconfig(struct ice_softc *sc, struct ice_dcbx_cfg *old_cfg,
    struct ice_dcbx_cfg *new_cfg)
{
	bool needs_reconfig = false;

	/* No change detected in DCBX config */
	if (!memcmp(old_cfg, new_cfg, sizeof(*old_cfg))) {
		DNPRINTF(ICE_DBG_DCB,
		    "%s: No change detected in local DCBX configuration\n",
		    sc->sc_dev.dv_xname);
		return (false);
	}

	/* Check if ETS config has changed */
	if (memcmp(&new_cfg->etscfg, &old_cfg->etscfg,
		   sizeof(new_cfg->etscfg))) {
		/* If Priority Table has changed, driver reconfig is needed */
		if (memcmp(&new_cfg->etscfg.prio_table,
			   &old_cfg->etscfg.prio_table,
			   sizeof(new_cfg->etscfg.prio_table))) {
			DNPRINTF(ICE_DBG_DCB, "%s: ETS UP2TC changed\n",
			    __func__);
			needs_reconfig = true;
		}

		/* These are just informational */
		if (memcmp(&new_cfg->etscfg.tcbwtable,
			   &old_cfg->etscfg.tcbwtable,
			   sizeof(new_cfg->etscfg.tcbwtable))) {
			DNPRINTF(ICE_DBG_DCB, "%s: ETS TCBW table changed\n",
			    __func__);
			needs_reconfig = true;
		}

		if (memcmp(&new_cfg->etscfg.tsatable,
			   &old_cfg->etscfg.tsatable,
			   sizeof(new_cfg->etscfg.tsatable))) {
			DNPRINTF(ICE_DBG_DCB, "%s: ETS TSA table changed\n",
			    __func__);
			needs_reconfig = true;
		}
	}

	/* Check if PFC config has changed */
	if (memcmp(&new_cfg->pfc, &old_cfg->pfc, sizeof(new_cfg->pfc))) {
		DNPRINTF(ICE_DBG_DCB, "%s: PFC config changed\n", __func__);
		needs_reconfig = true;
	}

	/* Check if APP table has changed */
	if (memcmp(&new_cfg->app, &old_cfg->app, sizeof(new_cfg->app)))
		DNPRINTF(ICE_DBG_DCB, "%s: APP Table changed\n", __func__);

	DNPRINTF(ICE_DBG_DCB, "%s result: %d\n", __func__, needs_reconfig);

	return (needs_reconfig);
}

/**
 * ice_do_dcb_reconfig - notify RDMA and reconfigure PF LAN VSI
 * @sc: the device private softc
 * @pending_mib: FW has a pending MIB change to execute
 * 
 * @pre Determined that the DCB configuration requires a change
 *
 * Reconfigures the PF LAN VSI based on updated DCB configuration
 * found in the hw struct's/port_info's/ local dcbx configuration.
 */
void
ice_do_dcb_reconfig(struct ice_softc *sc, bool pending_mib)
{
#if 0
	struct ice_aqc_port_ets_elem port_ets = { 0 };
	struct ice_dcbx_cfg *local_dcbx_cfg;
	struct ice_hw *hw = &sc->hw;
	struct ice_port_info *pi;
	device_t dev = sc->dev;
	enum ice_status status;

	pi = sc->hw.port_info;
	local_dcbx_cfg = &pi->qos_cfg.local_dcbx_cfg;

	ice_rdma_notify_dcb_qos_change(sc);
	/* If there's a pending MIB, tell the FW to execute the MIB change
	 * now.
	 */
	if (pending_mib) {
		status = ice_lldp_execute_pending_mib(hw);
		if ((status == ICE_ERR_AQ_ERROR) &&
		    (hw->adminq.sq_last_status == ICE_AQ_RC_ENOENT)) {
			device_printf(dev,
			    "Execute Pending LLDP MIB AQ call failed, no pending MIB\n");
		} else if (status) {
			device_printf(dev,
			    "Execute Pending LLDP MIB AQ call failed, err %s aq_err %s\n",
			    ice_status_str(status),
			    ice_aq_str(hw->adminq.sq_last_status));
			/* This won't break traffic, but QoS will not work as expected */
		}
	}

	/* Set state when there's more than one TC */
	if (ice_dcb_get_num_tc(local_dcbx_cfg) > 1) {
		device_printf(dev, "Multiple traffic classes enabled\n");
		ice_set_state(&sc->state, ICE_STATE_MULTIPLE_TCS);
	} else {
		device_printf(dev, "Multiple traffic classes disabled\n");
		ice_clear_state(&sc->state, ICE_STATE_MULTIPLE_TCS);
	}

	/* Disable PF VSI since it's going to be reconfigured */
	ice_stop_pf_vsi(sc);

	/* Query ETS configuration and update SW Tx scheduler info */
	status = ice_query_port_ets(pi, &port_ets, sizeof(port_ets), NULL);
	if (status != ICE_SUCCESS) {
		device_printf(dev,
		    "Query Port ETS AQ call failed, err %s aq_err %s\n",
		    ice_status_str(status),
		    ice_aq_str(hw->adminq.sq_last_status));
		/* This won't break traffic, but QoS will not work as expected */
	}

	/* Change PF VSI configuration */
	ice_dcb_recfg(sc);

	/* Send new configuration to RDMA client driver */
	ice_rdma_dcb_qos_update(sc, pi);

	ice_request_stack_reinit(sc);
#else
	printf("%s: not implemented", __func__);
#endif
}

/**
 * ice_handle_mib_change_event - handle LLDP MIB change events
 * @sc: the device private softc
 * @event: event received on a control queue
 *
 * Checks the updated MIB it receives and possibly reconfigures the PF LAN
 * VSI depending on what has changed. This will also print out some debug
 * information about the MIB event if ICE_DBG_DCB is enabled in the debug_mask.
 */
void
ice_handle_mib_change_event(struct ice_softc *sc,
    struct ice_rq_event_info *event)
{
	struct ice_aqc_lldp_get_mib *params =
	    (struct ice_aqc_lldp_get_mib *)&event->desc.params.lldp_get_mib;
	struct ice_dcbx_cfg tmp_dcbx_cfg, *local_dcbx_cfg;
	struct ice_port_info *pi;
	struct ice_hw *hw = &sc->hw;
	bool needs_reconfig, mib_is_pending;
	enum ice_status status;
	uint8_t mib_type, bridge_type;
#if 0
	ASSERT_CFG_LOCKED(sc);
#endif
	ice_debug_print_mib_change_event(sc, event);

	pi = sc->hw.port_info;

	mib_type = (params->type & ICE_AQ_LLDP_MIB_TYPE_M) >>
	    ICE_AQ_LLDP_MIB_TYPE_S;
	bridge_type = (params->type & ICE_AQ_LLDP_BRID_TYPE_M) >>
	    ICE_AQ_LLDP_BRID_TYPE_S;
	mib_is_pending = (params->state & ICE_AQ_LLDP_MIB_CHANGE_STATE_M) >>
	    ICE_AQ_LLDP_MIB_CHANGE_STATE_S;

	/* Ignore if event is not for Nearest Bridge */
	if (bridge_type != ICE_AQ_LLDP_BRID_TYPE_NEAREST_BRID)
		return;

	/* Check MIB Type and return if event for Remote MIB update */
	if (mib_type == ICE_AQ_LLDP_MIB_REMOTE) {
		/* Update the cached remote MIB and return */
		status = ice_aq_get_dcb_cfg(pi->hw, ICE_AQ_LLDP_MIB_REMOTE,
					 ICE_AQ_LLDP_BRID_TYPE_NEAREST_BRID,
					 &pi->qos_cfg.remote_dcbx_cfg);
		if (status)
			printf("%s: Failed to get Remote DCB config; "
			    "status %s, aq_err %s\n",
			    sc->sc_dev.dv_xname, ice_status_str(status),
			    ice_aq_str(hw->adminq.sq_last_status));
		/* Not fatal if this fails */
		return;
	}

	/* Save line length by aliasing the local dcbx cfg */
	local_dcbx_cfg = &pi->qos_cfg.local_dcbx_cfg;
	/* Save off the old configuration and clear current config */
	tmp_dcbx_cfg = *local_dcbx_cfg;
	memset(local_dcbx_cfg, 0, sizeof(*local_dcbx_cfg));

	/* Update the current local_dcbx_cfg with new data */
	if (mib_is_pending) {
		ice_get_dcb_cfg_from_mib_change(pi, event);
	} else {
		/* Get updated DCBX data from firmware */
		status = ice_get_dcb_cfg(pi);
		if (status) {
			printf("%s: Failed to get Local DCB config; "
			    "status %s, aq_err %s\n",
			    sc->sc_dev.dv_xname, ice_status_str(status),
			    ice_aq_str(hw->adminq.sq_last_status));
			return;
		}
	}

	/* Check to see if DCB needs reconfiguring */
	needs_reconfig = ice_dcb_needs_reconfig(sc, &tmp_dcbx_cfg,
	    local_dcbx_cfg);

	if (!needs_reconfig && !mib_is_pending)
		return;

	/* Reconfigure -- this will also notify FW that configuration is done,
	 * if the FW MIB change is only pending instead of executed.
	 */
	ice_do_dcb_reconfig(sc, mib_is_pending);
}

/**
 * ice_handle_lan_overflow_event - helper function to log LAN overflow events
 * @sc: device softc
 * @event: event received on a control queue
 *
 * Prints out a message when a LAN overflow event is detected on a receive
 * queue.
 */
void
ice_handle_lan_overflow_event(struct ice_softc *sc,
    struct ice_rq_event_info *event)
{
#ifdef ICE_DEBUG
	struct ice_aqc_event_lan_overflow *params =
	    (struct ice_aqc_event_lan_overflow *)&event->desc.params.lan_overflow;

	DNPRINTF(ICE_DBG_DCB, "%s: LAN overflow event detected, "
	    "prtdcb_ruptq=0x%08x, qtx_ctl=0x%08x\n",
	    sc->sc_dev.dv_xname, le32toh(params->prtdcb_ruptq),
	    le32toh(params->qtx_ctl));
#endif
}

/**
 * ice_print_health_status_string - Print message for given FW health event
 * @dev: the PCIe device
 * @elem: health status element containing status code
 *
 * A rather large list of possible health status codes and their associated
 * messages.
 */
void
ice_print_health_status_string(struct ice_softc *sc,
    struct ice_aqc_health_status_elem *elem)
{
	uint16_t status_code = le16toh(elem->health_status_code);

	switch (status_code) {
	case ICE_AQC_HEALTH_STATUS_INFO_RECOVERY:
		printf("%s: The device is in firmware recovery mode.\n",
		    sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_ERR_FLASH_ACCESS:
		printf("%s: The flash chip cannot be accessed.\n",
		    sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_ERR_NVM_AUTH:
		printf("%s: NVM authentication failed.\n",
		    sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_ERR_OROM_AUTH:
		printf("%s: Option ROM authentication failed.\n",
		    sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_ERR_DDP_AUTH:
		printf("%s: DDP package failed.\n", sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_ERR_NVM_COMPAT:
		printf("%s: NVM image is incompatible.\n", sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_ERR_OROM_COMPAT:
		printf("%s: Option ROM is incompatible.\n", sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_ERR_DCB_MIB:
		printf("%s: Supplied MIB file is invalid. "
		    "DCB reverted to default configuration.\n",
		    sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_ERR_UNKNOWN_MOD_STRICT:
		printf("%s: An unsupported module was detected.\n",
		    sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_ERR_MOD_TYPE:
		printf("%s: Module type is not supported.\n",
		    sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_ERR_MOD_QUAL:
		printf("%s: Module is not qualified.\n", sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_ERR_MOD_COMM:
		printf("%s: Device cannot communicate with the module.\n",
		    sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_ERR_MOD_CONFLICT:
		printf("%s: Unresolved module conflict.\n",
		    sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_ERR_MOD_NOT_PRESENT:
		printf("%s: Module is not present.\n", sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_INFO_MOD_UNDERUTILIZED:
		printf("%s: Underutilized module.\n", sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_ERR_UNKNOWN_MOD_LENIENT:
		printf("%s: An unsupported module was detected.\n",
		    sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_ERR_INVALID_LINK_CFG:
		printf("%s: Invalid link configuration.\n",
		    sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_ERR_PORT_ACCESS:
		printf("%s: Port hardware access error.\n",
		    sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_ERR_PORT_UNREACHABLE:
		printf("%s: A port is unreachable.\n", sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_INFO_PORT_SPEED_MOD_LIMITED:
		printf("%s: Port speed is limited due to module.\n",
		    sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_ERR_PARALLEL_FAULT:
		printf("%s: A parallel fault was detected.\n",
		    sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_INFO_PORT_SPEED_PHY_LIMITED:
		printf("%s: Port speed is limited by PHY capabilities.\n",
		    sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_ERR_NETLIST_TOPO:
		printf("%s: LOM topology netlist is corrupted.\n",
		    sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_ERR_NETLIST:
		printf("%s: Unrecoverable netlist error.\n",
		    sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_ERR_TOPO_CONFLICT:
		printf("%s: Port topology conflict.\n", sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_ERR_LINK_HW_ACCESS:
		printf("%s: Unrecoverable hardware access error.\n",
		    sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_ERR_LINK_RUNTIME:
		printf("%s: Unrecoverable runtime error.\n",
		    sc->sc_dev.dv_xname);
		break;
	case ICE_AQC_HEALTH_STATUS_ERR_DNL_INIT:
		printf("%s: Link management engine failed to initialize.\n",
		    sc->sc_dev.dv_xname);
		break;
	default:
		break;
	}
}

/**
 * ice_handle_health_status_event - helper function to output health status
 * @sc: device softc structure
 * @event: event received on a control queue
 *
 * Prints out the appropriate string based on the given Health Status Event
 * code.
 */
void
ice_handle_health_status_event(struct ice_softc *sc,
			       struct ice_rq_event_info *event)
{
	struct ice_aqc_health_status_elem *health_info;
	uint16_t status_count;
	int i;

	if (!ice_is_bit_set(sc->feat_en, ICE_FEATURE_HEALTH_STATUS))
		return;

	health_info = (struct ice_aqc_health_status_elem *)event->msg_buf;
	status_count = le16toh(
	    event->desc.params.get_health_status.health_status_count);

	if (status_count > (event->buf_len / sizeof(*health_info))) {
		DPRINTF("%s: Received a health status event with invalid "
		    "event count\n", sc->sc_dev.dv_xname);
		return;
	}

	for (i = 0; i < status_count; i++) {
		ice_print_health_status_string(sc, health_info);
		health_info++;
	}
}
/**
 * ice_process_ctrlq_event - Respond to a controlq event
 * @sc: device private structure
 * @qname: the name for this controlq
 * @event: the event to process
 *
 * Perform actions in response to various controlq event notifications.
 */
void
ice_process_ctrlq_event(struct ice_softc *sc, const char *qname,
			struct ice_rq_event_info *event)
{
	uint16_t opcode;

	opcode = le16toh(event->desc.opcode);

	switch (opcode) {
	case ice_aqc_opc_get_link_status:
		ice_process_link_event(sc, event);
		break;
	case ice_aqc_opc_fw_logs_event:
		ice_handle_fw_log_event(sc, &event->desc, event->msg_buf);
		break;
	case ice_aqc_opc_lldp_set_mib_change:
		ice_handle_mib_change_event(sc, event);
		break;
	case ice_aqc_opc_event_lan_overflow:
		ice_handle_lan_overflow_event(sc, event);
		break;
	case ice_aqc_opc_get_health_status:
		ice_handle_health_status_event(sc, event);
		break;
	default:
		printf("%s: %s Receive Queue unhandled event 0x%04x ignored\n",
		    sc->sc_dev.dv_xname, qname, opcode);
		break;
	}
}

/**
 * ice_clean_rq_elem
 * @hw: pointer to the HW struct
 * @cq: pointer to the specific Control queue
 * @e: event info from the receive descriptor, includes any buffers
 * @pending: number of events that could be left to process
 *
 * Clean one element from the receive side of a control queue. On return 'e'
 * contains contents of the message, and 'pending' contains the number of
 * events left to process.
 */
enum ice_status
ice_clean_rq_elem(struct ice_hw *hw, struct ice_ctl_q_info *cq,
		  struct ice_rq_event_info *e, uint16_t *pending)
{
	uint16_t ntc = cq->rq.next_to_clean;
	enum ice_aq_err rq_last_status;
	enum ice_status ret_code = ICE_SUCCESS;
	struct ice_aq_desc *desc;
	struct ice_dma_mem *bi;
	uint16_t desc_idx;
	uint16_t datalen;
	uint16_t flags;
	uint16_t ntu;

	/* pre-clean the event info */
	memset(&e->desc, 0, sizeof(e->desc));
#if 0
	/* take the lock before we start messing with the ring */
	ice_acquire_lock(&cq->rq_lock);
#endif
	if (!cq->rq.count) {
		DNPRINTF(ICE_DBG_AQ_MSG,
		    "%s: Control Receive queue not initialized.\n", __func__);
		ret_code = ICE_ERR_AQ_EMPTY;
		goto clean_rq_elem_err;
	}

	/* set next_to_use to head */
	ntu = (uint16_t)(ICE_READ(hw, cq->rq.head) & cq->rq.head_mask);

	if (ntu == ntc) {
		/* nothing to do - shouldn't need to update ring's values */
		ret_code = ICE_ERR_AQ_NO_WORK;
		goto clean_rq_elem_out;
	}

	/* now clean the next descriptor */
	desc = ICE_CTL_Q_DESC(cq->rq, ntc);
	desc_idx = ntc;

	rq_last_status = (enum ice_aq_err)le16toh(desc->retval);
	flags = le16toh(desc->flags);
	if (flags & ICE_AQ_FLAG_ERR) {
		ret_code = ICE_ERR_AQ_ERROR;
		DNPRINTF(ICE_DBG_AQ_MSG, "%s: Control Receive Queue "
		    "Event 0x%04X received with error 0x%X\n",
		    __func__, le16toh(desc->opcode), rq_last_status);
	}
	memcpy(&e->desc, desc, sizeof(e->desc));
	datalen = le16toh(desc->datalen);
	e->msg_len = MIN(datalen, e->buf_len);
	if (e->msg_buf && e->msg_len)
		memcpy(e->msg_buf, cq->rq.r.rq_bi[desc_idx].va, e->msg_len);

	DNPRINTF(ICE_DBG_AQ_DESC, "%s: ARQ: desc and buffer:\n", __func__);
	ice_debug_cq(hw, cq, (void *)desc, e->msg_buf, cq->rq_buf_size, true);

	/* Restore the original datalen and buffer address in the desc,
	 * FW updates datalen to indicate the event message size
	 */
	bi = &cq->rq.r.rq_bi[ntc];
	memset(desc, 0, sizeof(*desc));

	desc->flags = htole16(ICE_AQ_FLAG_BUF);
	if (cq->rq_buf_size > ICE_AQ_LG_BUF)
		desc->flags |= htole16(ICE_AQ_FLAG_LB);
	desc->datalen = htole16(bi->size);
	desc->params.generic.addr_high = htole32(ICE_HI_DWORD(bi->pa));
	desc->params.generic.addr_low = htole32(ICE_LO_DWORD(bi->pa));

	/* set tail = the last cleaned desc index. */
	ICE_WRITE(hw, cq->rq.tail, ntc);
	/* ntc is updated to tail + 1 */
	ntc++;
	if (ntc == cq->num_rq_entries)
		ntc = 0;
	cq->rq.next_to_clean = ntc;
	cq->rq.next_to_use = ntu;

clean_rq_elem_out:
	/* Set pending if needed, unlock and return */
	if (pending) {
		/* re-read HW head to calculate actual pending messages */
		ntu = (uint16_t)(ICE_READ(hw, cq->rq.head) & cq->rq.head_mask);
		*pending = (uint16_t)((ntc > ntu ? cq->rq.count : 0) +
		    (ntu - ntc));
	}
clean_rq_elem_err:
#if 0
	ice_release_lock(&cq->rq_lock);
#endif
	return ret_code;
}

/**
 * ice_process_ctrlq - helper function to process controlq rings
 * @sc: device private structure
 * @q_type: specific control queue type
 * @pending: return parameter to track remaining events
 *
 * Process controlq events for a given control queue type. Returns zero on
 * success, and an error code on failure. If successful, pending is the number
 * of remaining events left in the queue.
 */
int
ice_process_ctrlq(struct ice_softc *sc, enum ice_ctl_q q_type,
    uint16_t *pending)
{
	struct ice_rq_event_info event = { { 0 } };
	struct ice_hw *hw = &sc->hw;
	struct ice_ctl_q_info *cq;
	enum ice_status status;
	const char *qname;
	int loop = 0;

	switch (q_type) {
	case ICE_CTL_Q_ADMIN:
		cq = &hw->adminq;
		qname = "Admin";
		break;
	case ICE_CTL_Q_MAILBOX:
		cq = &hw->mailboxq;
		qname = "Mailbox";
		break;
	default:
		DPRINTF("%s: Unknown control queue type 0x%x\n",
		    __func__, q_type);
		return 0;
	}

	ice_check_ctrlq_errors(sc, qname, cq);

	/*
	 * Control queue processing happens during the admin task which may be
	 * holding a non-sleepable lock, so we *must* use M_NOWAIT here.
	 */
	event.buf_len = cq->rq_buf_size;
	event.msg_buf = (uint8_t *)malloc(event.buf_len, M_DEVBUF,
	    M_ZERO | M_NOWAIT);
	if (!event.msg_buf) {
		printf("%s: Unable to allocate memory for %s Receive Queue "
		    "event\n", sc->sc_dev.dv_xname, qname);
		return (ENOMEM);
	}

	do {
		status = ice_clean_rq_elem(hw, cq, &event, pending);
		if (status == ICE_ERR_AQ_NO_WORK)
			break;
		if (status) {
			if (q_type == ICE_CTL_Q_ADMIN) {
				printf("%s: %s Receive Queue event error %s\n",
				    sc->sc_dev.dv_xname, qname,
				    ice_status_str(status));
			} else {
				printf("%s: %s Receive Queue event error %s\n",
				    sc->sc_dev.dv_xname, qname,
				    ice_status_str(status));
			}
			free(event.msg_buf, M_DEVBUF, event.buf_len);
			return (EIO);
		}
		/* XXX should we separate this handler by controlq type? */
		ice_process_ctrlq_event(sc, qname, &event);
	} while (*pending && (++loop < ICE_CTRLQ_WORK_LIMIT));

	free(event.msg_buf, M_DEVBUF, event.buf_len);

	return 0;
}

/**
 * ice_poll_for_media_avail - Re-enable link if media is detected
 * @sc: device private structure
 *
 * Intended to be called from the driver's timer function, this function
 * sends the Get Link Status AQ command and re-enables HW link if the
 * command says that media is available.
 *
 * If the driver doesn't have the "NO_MEDIA" state set, then this does nothing,
 * since media removal events are supposed to be sent to the driver through
 * a link status event.
 */
void
ice_poll_for_media_avail(struct ice_softc *sc)
{
	struct ice_hw *hw = &sc->hw;
	struct ifnet *ifp = &sc->sc_ac.ac_if;
	struct ice_port_info *pi = hw->port_info;

	if (ice_test_state(&sc->state, ICE_STATE_NO_MEDIA)) {
		pi->phy.get_link_info = true;
		ice_get_link_status(pi, &sc->link_up);

		if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) {
			enum ice_status status;

			/* Re-enable link and re-apply user link settings */
			if (ice_test_state(&sc->state,
			    ICE_STATE_LINK_ACTIVE_ON_DOWN) ||
			    (ifp->if_flags & IFF_UP)) {
				ice_apply_saved_phy_cfg(sc,
				    ICE_APPLY_LS_FEC_FC);

				/* Update OS with changes in media capability */
				status = ice_add_media_types(sc, &sc->media);
				if (status) {
					printf("%s: Error adding device "
					    "media types: %s aq_err %s\n",
					    sc->sc_dev.dv_xname,
					    ice_status_str(status),
					    ice_aq_str(
					    hw->adminq.sq_last_status));
				}
			}

			ice_clear_state(&sc->state, ICE_STATE_NO_MEDIA);
		}
	}
}

/**
 * ice_if_update_admin_status - update admin status
 * @ctx: iflib ctx structure
 *
 * Called by iflib to update the admin status. For our purposes, this means
 * check the adminq, and update the link status. It's ultimately triggered by
 * our admin interrupt, or by the ice_if_timer periodically.
 *
 * @pre assumes the caller holds the iflib CTX lock
 */
void
ice_if_update_admin_status(void *arg)
{
	struct ice_softc *sc = (struct ice_softc *)arg;
	enum ice_fw_modes fw_mode;
	bool reschedule = false;
	uint16_t pending = 0;
	int s;
#if 0
	ASSERT_CTX_LOCKED(sc);
#endif
	s = splnet();

	if (ice_driver_is_detaching(sc)) {
		splx(s);
		return;
	}

	/* Check if the firmware entered recovery mode at run time */
	fw_mode = ice_get_fw_mode(&sc->hw);
	if (fw_mode == ICE_FW_MODE_REC) {
		if (!ice_testandset_state(&sc->state,
		    ICE_STATE_RECOVERY_MODE)) {
			/* If we just entered recovery mode, log a warning to
			 * the system administrator and deinit driver state
			 * that is no longer functional.
			 */
			ice_transition_recovery_mode(sc);
		}
	} else if (fw_mode == ICE_FW_MODE_ROLLBACK) {
		if (!ice_testandset_state(&sc->state,
		    ICE_STATE_ROLLBACK_MODE)) {
			/* Rollback mode isn't fatal, but we don't want to
			 * repeatedly post a message about it.
			 */
			ice_print_rollback_msg(&sc->hw);
		}
	}

	/* Handle global reset events */
	ice_handle_reset_event(sc);

	/* Handle PF reset requests */
	ice_handle_pf_reset_request(sc);

	/* Handle MDD events */
	ice_handle_mdd_event(sc);

	if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED) ||
	    ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET) ||
	    ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) {
		/*
		 * If we know the control queues are disabled, skip processing
		 * the control queues entirely.
		 */
		;
	} else if (ice_testandclear_state(&sc->state,
	    ICE_STATE_CONTROLQ_EVENT_PENDING)) {
		ice_process_ctrlq(sc, ICE_CTL_Q_ADMIN, &pending);
		if (pending > 0)
			reschedule = true;

		ice_process_ctrlq(sc, ICE_CTL_Q_MAILBOX, &pending);
		if (pending > 0)
			reschedule = true;
	}

	/* Poll for link up */
	ice_poll_for_media_avail(sc);

	/* Check and update link status */
	ice_update_link_status(sc, false);

	/*
	 * If there are still messages to process, we need to reschedule
	 * ourselves. Otherwise, we can just re-enable the interrupt. We'll be
	 * woken up at the next interrupt or timer event.
	 */
	if (reschedule) {
		ice_set_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING);
		task_add(systq, &sc->sc_admin_task);
	} else {
		ice_enable_intr(&sc->hw, 0);
	}

	splx(s);
}


/**
 * ice_admin_timer - called periodically to trigger the admin task
 * @arg: timeout(9) argument pointing to the device private softc structure
 *
 * Timer function used as part of a timeout(9) timer that will periodically
 * trigger the admin task, even when the interface is down.
 *
 * @remark because this is a timeout function, it cannot sleep and should not
 * attempt taking the iflib CTX lock.
 */
void
ice_admin_timer(void *arg)
{
	struct ice_softc *sc = (struct ice_softc *)arg;
	struct ifnet *ifp = &sc->sc_ac.ac_if;
	int s;

	s = splnet();

	if (ice_driver_is_detaching(sc) ||
	    (ifp->if_flags & IFF_RUNNING) == 0) {
		splx(s);
		return;
	}

	/* Fire off the admin task */
	task_add(systq, &sc->sc_admin_task);

	/* Reschedule the admin timer */
	timeout_add_nsec(&sc->sc_admin_timer, SEC_TO_NSEC(1));

	splx(s);
}

/**
 * @enum hmc_error_type
 * @brief enumeration of HMC errors
 *
 * Enumeration defining the possible HMC errors that might occur.
 */
enum hmc_error_type {
	HMC_ERR_PMF_INVALID = 0,
	HMC_ERR_VF_IDX_INVALID = 1,
	HMC_ERR_VF_PARENT_PF_INVALID = 2,
	/* 3 is reserved */
	HMC_ERR_INDEX_TOO_BIG = 4,
	HMC_ERR_ADDRESS_TOO_LARGE = 5,
	HMC_ERR_SEGMENT_DESC_INVALID = 6,
	HMC_ERR_SEGMENT_DESC_TOO_SMALL = 7,
	HMC_ERR_PAGE_DESC_INVALID = 8,
	HMC_ERR_UNSUPPORTED_REQUEST_COMPLETION = 9,
	/* 10 is reserved */
	HMC_ERR_INVALID_OBJECT_TYPE = 11,
	/* 12 is reserved */
};

/**
 * ice_log_hmc_error - Log an HMC error message
 * @hw: device hw structure
 * @dev: the device to pass to device_printf()
 *
 * Log a message when an HMC error interrupt is triggered.
 */
void
ice_log_hmc_error(struct ice_hw *hw)
{
	struct ice_softc *sc = hw->hw_sc;
	uint32_t info, data;
	uint8_t index, errtype, objtype;
	bool isvf;

	info = ICE_READ(hw, PFHMC_ERRORINFO);
	data = ICE_READ(hw, PFHMC_ERRORDATA);

	index = (uint8_t)(info & PFHMC_ERRORINFO_PMF_INDEX_M);
	errtype = (uint8_t)((info & PFHMC_ERRORINFO_HMC_ERROR_TYPE_M) >>
		       PFHMC_ERRORINFO_HMC_ERROR_TYPE_S);
	objtype = (uint8_t)((info & PFHMC_ERRORINFO_HMC_OBJECT_TYPE_M) >>
		       PFHMC_ERRORINFO_HMC_OBJECT_TYPE_S);

	isvf = info & PFHMC_ERRORINFO_PMF_ISVF_M;

	printf("%s: %s HMC Error detected on PMF index %d: "
	    "error type %d, object type %d, data 0x%08x\n",
	    sc->sc_dev.dv_xname, isvf ? "VF" : "PF", index,
	    errtype, objtype, data);

	switch (errtype) {
	case HMC_ERR_PMF_INVALID:
		DPRINTF("Private Memory Function is not valid\n");
		break;
	case HMC_ERR_VF_IDX_INVALID:
		DPRINTF("Invalid Private Memory Function index for PE enabled VF\n");
		break;
	case HMC_ERR_VF_PARENT_PF_INVALID:
		DPRINTF("Invalid parent PF for PE enabled VF\n");
		break;
	case HMC_ERR_INDEX_TOO_BIG:
		DPRINTF("Object index too big\n");
		break;
	case HMC_ERR_ADDRESS_TOO_LARGE:
		DPRINTF("Address extends beyond segment descriptor limit\n");
		break;
	case HMC_ERR_SEGMENT_DESC_INVALID:
		DPRINTF("Segment descriptor is invalid\n");
		break;
	case HMC_ERR_SEGMENT_DESC_TOO_SMALL:
		DPRINTF("Segment descriptor is too small\n");
		break;
	case HMC_ERR_PAGE_DESC_INVALID:
		DPRINTF("Page descriptor is invalid\n");
		break;
	case HMC_ERR_UNSUPPORTED_REQUEST_COMPLETION:
		DPRINTF("Unsupported Request completion received from PCIe\n");
		break;
	case HMC_ERR_INVALID_OBJECT_TYPE:
		DPRINTF("Invalid object type\n");
		break;
	default:
		DPRINTF("Unknown HMC error\n");
	}

	/* Clear the error indication */
	ICE_WRITE(hw, PFHMC_ERRORINFO, 0);
}

/**
 * Interrupt handler for MSI-X admin interrupt
 */
int
ice_intr0(void *xsc)
{
	struct ice_softc *sc = (struct ice_softc *)xsc;
	struct ice_hw *hw = &sc->hw;
	struct ifnet *ifp = &sc->sc_ac.ac_if;
	uint32_t oicr;

	/* There is no safe way to modify the enabled miscellaneous causes of
	 * the OICR vector at runtime, as doing so would be prone to race
	 * conditions. Reading PFINT_OICR will unmask the associated interrupt
	 * causes and allow future interrupts to occur. The admin interrupt
	 * vector will not be re-enabled until after we exit this function,
	 * but any delayed tasks must be resilient against possible "late
	 * arrival" interrupts that occur while we're already handling the
	 * task. This is done by using state bits and serializing these
	 * delayed tasks via the admin status task function.
	 */
	oicr = ICE_READ(hw, PFINT_OICR);

	/* Processing multiple controlq interrupts on a single vector does not
	 * provide an indication of which controlq triggered the interrupt.
	 * We might try reading the INTEVENT bit of the respective PFINT_*_CTL
	 * registers. However, the INTEVENT bit is not guaranteed to be set as
	 * it gets automatically cleared when the hardware acknowledges the
	 * interrupt.
	 *
	 * This means we don't really have a good indication of whether or
	 * which controlq triggered this interrupt. We'll just notify the
	 * admin task that it should check all the controlqs.
	 */
	ice_set_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING);

	if (oicr & PFINT_OICR_VFLR_M) {
		ice_set_state(&sc->state, ICE_STATE_VFLR_PENDING);
	}

	if (oicr & PFINT_OICR_MAL_DETECT_M) {
		ice_set_state(&sc->state, ICE_STATE_MDD_PENDING);
	}

	if (oicr & PFINT_OICR_GRST_M) {
		uint32_t reset;

		reset = (ICE_READ(hw, GLGEN_RSTAT) &
		    GLGEN_RSTAT_RESET_TYPE_M) >> GLGEN_RSTAT_RESET_TYPE_S;
#if 0
		if (reset == ICE_RESET_CORER)
			sc->soft_stats.corer_count++;
		else if (reset == ICE_RESET_GLOBR)
			sc->soft_stats.globr_count++;
		else
			sc->soft_stats.empr_count++;
#endif
		/* There are a couple of bits at play for handling resets.
		 * First, the ICE_STATE_RESET_OICR_RECV bit is used to
		 * indicate that the driver has received an OICR with a reset
		 * bit active, indicating that a CORER/GLOBR/EMPR is about to
		 * happen. Second, we set hw->reset_ongoing to indicate that
		 * the hardware is in reset. We will set this back to false as
		 * soon as the driver has determined that the hardware is out
		 * of reset.
		 *
		 * If the driver wishes to trigger a request, it can set one of
		 * the ICE_STATE_RESET_*_REQ bits, which will trigger the
		 * correct type of reset.
		 */
		if (!ice_testandset_state(&sc->state,
		    ICE_STATE_RESET_OICR_RECV)) {
			hw->reset_ongoing = true;
			/*
			 * During the NVM update process, there is a driver
			 * reset and link goes down and then up. The below
			 * if-statement prevents a second link flap from
			 * occurring in ice_up().
			 */
			if (ifp->if_flags & IFF_UP) {
				ice_set_state(&sc->state,
				    ICE_STATE_FIRST_INIT_LINK);
			}
		}
	}

	if (oicr & PFINT_OICR_ECC_ERR_M) {
		DPRINTF("%s: ECC Error detected!\n", sc->sc_dev.dv_xname);
		ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
	}

	if (oicr & (PFINT_OICR_PE_CRITERR_M | PFINT_OICR_HMC_ERR_M)) {
		if (oicr & PFINT_OICR_HMC_ERR_M)
			/* Log the HMC errors */
			ice_log_hmc_error(hw);
#if 0
		ice_rdma_notify_pe_intr(sc, oicr);
#endif
	}

	if (oicr & PFINT_OICR_PCI_EXCEPTION_M) {
		DPRINTF("%s: PCI Exception detected!\n", sc->sc_dev.dv_xname);
		ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
	}

	task_add(systq, &sc->sc_admin_task);
	return 1;
}

/*
 * Macro to help extract the NIC mode flexible Rx descriptor fields from the
 * advanced 32byte Rx descriptors.
 */
#define ICE_RX_FLEX_NIC(desc, field) \
	(((struct ice_32b_rx_flex_desc_nic *)desc)->field)

void
ice_rx_checksum(struct mbuf *m, uint16_t status0)
{
	/* TODO */
}

int
ice_rxeof(struct ice_softc *sc, struct ice_rx_queue *rxq)
{
	struct ifiqueue *ifiq = rxq->rxq_ifiq;
	struct ifnet *ifp = &sc->sc_ac.ac_if;
	union ice_32b_rx_flex_desc *ring, *cur;
	struct ice_rx_map *rxm;
	bus_dmamap_t map;
	unsigned int cons, prod;
	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
	struct mbuf *m;
	uint16_t status0;
	unsigned int eop;
	unsigned int len;
	unsigned int mask;
	int done = 0;

	prod = rxq->rxq_prod;
	cons = rxq->rxq_cons;

	if (cons == prod)
		return (0);

	rxm = &rxq->rx_map[cons];
	map = rxm->rxm_map;
	bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
	    BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);

	ring = ICE_DMA_KVA(&rxq->rx_desc_mem);
	mask = rxq->desc_count - 1;

	do {
		cur = &ring[cons];

		status0 = le16toh(cur->wb.status_error0);
		if ((status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S)) == 0)
			break;

		if_rxr_put(&rxq->rxq_acct, 1);

		rxm = &rxq->rx_map[cons];

		map = rxm->rxm_map;
		bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
		    BUS_DMASYNC_POSTREAD);
		bus_dmamap_unload(sc->sc_dmat, map);
		
		m = rxm->rxm_m;
		rxm->rxm_m = NULL;

		len = le16toh(cur->wb.pkt_len) & ICE_RX_FLX_DESC_PKT_LEN_M;
		m->m_len = len;
		m->m_pkthdr.len = 0;

		m->m_next = NULL;
		*rxq->rxq_m_tail = m;
		rxq->rxq_m_tail = &m->m_next;

		m = rxq->rxq_m_head;
		m->m_pkthdr.len += len;

		eop = !!(status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_EOF_S));
		if (eop && (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_RXE_S))) {
			/*
			 * Make sure packets with bad L2 values are discarded.
			 * This bit is only valid in the last descriptor.
			 */
			ifp->if_ierrors++;
			m_freem(m);
			m = NULL;
			rxq->rxq_m_head = NULL;
			rxq->rxq_m_tail = &rxq->rxq_m_head;
		} else if (eop) {
#if NVLAN > 0
			if (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_L2TAG1P_S)) {
				m->m_pkthdr.ether_vtag =
				    le16toh(cur->wb.l2tag1);
				SET(m->m_flags, M_VLANTAG);
			}
#endif
			if (status0 &
			    BIT(ICE_RX_FLEX_DESC_STATUS0_RSS_VALID_S)) {
				m->m_pkthdr.ph_flowid = le32toh(
				    ICE_RX_FLEX_NIC(&cur->wb, rss_hash));
				m->m_pkthdr.csum_flags |= M_FLOWID;
			}

			ice_rx_checksum(m, status0);
			ml_enqueue(&ml, m);

			rxq->rxq_m_head = NULL;
			rxq->rxq_m_tail = &rxq->rxq_m_head;
		}

		cons++;
		cons &= mask;

		done = 1;
	} while (cons != prod);

	if (done) {
		rxq->rxq_cons = cons;
		if (ifiq_input(ifiq, &ml))
			if_rxr_livelocked(&rxq->rxq_acct);
		ice_rxfill(sc, rxq);
	}

	bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
	    BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);

	return (done);
}

int
ice_txeof(struct ice_softc *sc, struct ice_tx_queue *rxq)
{
	/* TODO */
	return 0;
}

int
ice_intr_vector(void *ivp)
{
	struct ice_intr_vector *iv = ivp;
	struct ice_softc *sc = iv->iv_sc;
	struct ifnet *ifp = &sc->sc_ac.ac_if;
	int rv = 0, v = iv->iv_qid + 1;

	if (ISSET(ifp->if_flags, IFF_RUNNING)) {
		rv |= ice_rxeof(sc, iv->iv_rxq);
		rv |= ice_txeof(sc, iv->iv_txq);
	}

	/* Wake threads waiting for software interrupt confirmation. */
	if (sc->sw_intr[v] == -1) {
		sc->sw_intr[v] = 1;
		wakeup(&sc->sw_intr[v]);
	}
		
	ice_enable_intr(&sc->hw, v);
	return rv;
}

/**
 * ice_allocate_msix - Allocate MSI-X vectors for the interface
 * @sc: the device private softc
 *
 * @post on success this function must set the following scctx parameters:
 * isc_vectors, isc_nrxqsets, isc_ntxqsets, and isc_intr.
 *
 * @returns zero on success or an error code on failure.
 */
int
ice_allocate_msix(struct ice_softc *sc)
{
	int err, i;

	sc->sc_ihc = pci_intr_establish(sc->sc_pct, sc->sc_ih,
	    IPL_NET | IPL_MPSAFE, ice_intr0, sc, sc->sc_dev.dv_xname);
	if (sc->sc_ihc == NULL) {
		printf("%s: unable to establish interrupt handler\n",
		    sc->sc_dev.dv_xname);
		return ENOTRECOVERABLE;
	}

	if (sc->sc_intrmap) {
		for (i = 0; i < sc->sc_nqueues; i++) {
			struct ice_intr_vector *iv = &sc->sc_vectors[i];
			int v = i + 1; /* 0 is used for adminq */

			iv->iv_sc = sc;
			iv->iv_qid = i;
			iv->iv_ihc = pci_intr_establish_cpu(sc->sc_pct, iv->ih,
			    IPL_NET | IPL_MPSAFE,
			    intrmap_cpu(sc->sc_intrmap, i),
			    ice_intr_vector, iv, iv->iv_name);
			if (iv->iv_ihc == NULL) {
				printf("%s: unable to establish interrupt %d\n",
				    sc->sc_dev.dv_xname, v);
				err = ENOTRECOVERABLE;
				goto disestablish;
			}
		}
	}

	sc->isc_vectors = sc->sc_nvectors;
	sc->isc_nrxqsets = sc->sc_nqueues;
	sc->isc_ntxqsets = sc->sc_nqueues;

	return 0;

disestablish:
	if (sc->sc_intrmap != NULL) {
		for (i = 0; i < sc->sc_nqueues; i++) {
			struct ice_intr_vector *iv = &sc->sc_vectors[i];
			if (iv->iv_ihc == NULL)
				continue;
			pci_intr_disestablish(sc->sc_pct, iv->iv_ihc);
		}
	}
	pci_intr_disestablish(sc->sc_pct, sc->sc_ihc);
	sc->sc_ihc = NULL;
	return err;
}

void
ice_free_tx_queues(struct ice_softc *sc)
{
	struct ice_vsi *vsi = &sc->pf_vsi;
	struct ice_tx_queue *txq;
	struct ice_tx_map *map;
	int i, j;

	for (i = 0, txq = vsi->tx_queues; i < sc->sc_nqueues; i++, txq++) {
		ice_free_dma_mem(&sc->hw, &txq->tx_desc_mem);
		for (j = 0; j < txq->desc_count; j++) {
			map = &txq->tx_map[j];
			if (map->txm_map != NULL) {
				bus_dmamap_destroy(sc->sc_dmat, map->txm_map);
				map->txm_map = NULL;
			}
		}
		free(txq->tx_map, M_DEVBUF, txq->desc_count * sizeof(*map));
		txq->tx_map = NULL;
		if (txq->tx_rsq != NULL) {
			free(txq->tx_rsq, M_DEVBUF,
			    sc->isc_ntxd[0] * sizeof(uint16_t));
			txq->tx_rsq = NULL;
		}
	}

	free(vsi->tx_queues, M_DEVBUF,
	    sc->sc_nqueues * sizeof(struct ice_tx_queue));
	vsi->tx_queues = NULL;
}

/* ice_tx_queues_alloc - Allocate Tx queue memory */
int
ice_tx_queues_alloc(struct ice_softc *sc)
{
	struct ice_vsi *vsi = &sc->pf_vsi;
	struct ice_tx_queue *txq;
	int err, i, j;

	KASSERT(sc->isc_ntxd[0] <= ICE_MAX_DESC_COUNT);
#if 0
	ASSERT_CTX_LOCKED(sc);
#endif
	/* Do not bother allocating queues if we're in recovery mode */
	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
		return (0);

	/* Allocate queue structure memory */
	if (!(vsi->tx_queues =
	      (struct ice_tx_queue *) mallocarray(sc->sc_nqueues,
	          sizeof(struct ice_tx_queue), M_DEVBUF, M_NOWAIT | M_ZERO))) {
		printf("%s: Unable to allocate Tx queue memory\n",
		    sc->sc_dev.dv_xname);
		return (ENOMEM);
	}

	/* Allocate Tx descriptor memory */
	for (i = 0, txq = vsi->tx_queues; i < sc->sc_nqueues; i++, txq++) {
		txq->tx_base = ice_alloc_dma_mem(&sc->hw, &txq->tx_desc_mem,
		    sc->isc_ntxd[i] * sizeof(struct ice_tx_desc));
		if (txq->tx_base == NULL) {
			printf("%s: Unable to allocate Tx descriptor memory\n",
			    sc->sc_dev.dv_xname);
			err = ENOMEM;
			goto free_tx_queues;
		}
		txq->tx_paddr = txq->tx_desc_mem.pa;
	}

	/* Create Tx queue DMA maps. */
	for (i = 0, txq = vsi->tx_queues; i < sc->sc_nqueues; i++, txq++) {
		struct ice_tx_map *map;
		int j;

		txq->tx_map = mallocarray(sc->isc_ntxd[i], sizeof(*map),
		    M_DEVBUF, M_NOWAIT| M_ZERO);
		if (txq->tx_map == NULL) {
			printf("%s: could not allocate Tx DMA map\n",
			    sc->sc_dev.dv_xname);
			err = ENOMEM;
			goto free_tx_queues;
		}

		for (j = 0; j < sc->isc_ntxd[i]; j++) {
			map = &txq->tx_map[j];
			if (bus_dmamap_create(sc->sc_dmat, ICE_MAX_FRAME_SIZE, 1,
			    ICE_MAX_FRAME_SIZE, 0,
			    BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT,
			    &map->txm_map) != 0) {
				printf("%s: could not allocate Tx DMA map\n",
				    sc->sc_dev.dv_xname);
				err = ENOMEM;
				goto free_tx_queues;
			}
		}
	}

	/* Allocate report status arrays */
	for (i = 0, txq = vsi->tx_queues; i < sc->sc_nqueues; i++, txq++) {
		if (!(txq->tx_rsq =
		      (uint16_t *) mallocarray(sc->isc_ntxd[0],
		          sizeof(uint16_t), M_DEVBUF, M_NOWAIT))) {
			printf("%s: Unable to allocate tx_rsq memory\n",
			    sc->sc_dev.dv_xname);
			err = ENOMEM;
			goto free_tx_queues;
		}
		/* Initialize report status array */
		for (j = 0; j < sc->isc_ntxd[i]; j++)
			txq->tx_rsq[j] = ICE_QIDX_INVALID;
	}

	/* Assign queues from PF space to the main VSI */
	err = ice_resmgr_assign_contiguous(&sc->tx_qmgr, vsi->tx_qmap,
	    sc->sc_nqueues);
	if (err) {
		printf("%s: Unable to assign PF queues: error %d\n",
		    sc->sc_dev.dv_xname, err);
		goto free_tx_queues;
	}
	vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS;
#if 0
	/* Add Tx queue sysctls context */
	ice_vsi_add_txqs_ctx(vsi);
#endif
	for (i = 0, txq = vsi->tx_queues; i < sc->sc_nqueues; i++, txq++) {
		/* q_handle == me when only one TC */
		txq->me = txq->q_handle = i;
		txq->vsi = vsi;

		/* store the queue size for easier access */
		txq->desc_count = sc->isc_ntxd[i];

		/* set doorbell address */
		txq->tail = QTX_COMM_DBELL(vsi->tx_qmap[i]);
#if 0
		ice_add_txq_sysctls(txq);
#endif
	}

	vsi->num_tx_queues = sc->sc_nqueues;

	return (0);

free_tx_queues:
	ice_free_tx_queues(sc);
	return err;
}

uint32_t
ice_hardmtu(struct ice_hw *hw)
{
	return hw->port_info->phy.link_info.max_frame_size -
	    ETHER_HDR_LEN - ETHER_CRC_LEN;
}

void
ice_free_rx_queues(struct ice_softc *sc)
{
	struct ice_vsi *vsi = &sc->pf_vsi;
	struct ice_rx_queue *rxq;
	struct ice_rx_map *map;
	int i, j;

	for (i = 0, rxq = vsi->rx_queues; i < sc->sc_nqueues; i++, rxq++) {
		ice_free_dma_mem(&sc->hw, &rxq->rx_desc_mem);
		for (j = 0; j < rxq->desc_count; j++) {
			map = &rxq->rx_map[j];
			if (map->rxm_map != NULL) {
				bus_dmamap_destroy(sc->sc_dmat, map->rxm_map);
				map->rxm_map = NULL;
			}
		}
		free(rxq->rx_map, M_DEVBUF, rxq->desc_count * sizeof(*map));
		rxq->rx_map = NULL;
	}

	free(vsi->rx_queues, M_DEVBUF,
	    sc->sc_nqueues * sizeof(struct ice_rx_queue));
	vsi->rx_queues = NULL;
}

void
ice_rxrefill(void *arg)
{
	struct ice_rx_queue *rxq = arg;
	struct ice_softc *sc = rxq->vsi->sc;

	ice_rxfill(sc, rxq);
}

/* ice_rx_queues_alloc - Allocate Rx queue memory */
int
ice_rx_queues_alloc(struct ice_softc *sc)
{
	struct ice_vsi *vsi = &sc->pf_vsi;
	struct ice_rx_queue *rxq;
	int err, i;
	uint32_t hardmtu = ice_hardmtu(&sc->hw);

	KASSERT(sc->isc_nrxd[0] <= ICE_MAX_DESC_COUNT);
#if 0
	ASSERT_CTX_LOCKED(sc);
#endif
	/* Do not bother allocating queues if we're in recovery mode */
	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
		return (0);

	/* Allocate queue structure memory */
	if (!(vsi->rx_queues =
	      (struct ice_rx_queue *) mallocarray(sc->sc_nqueues,
	          sizeof(struct ice_rx_queue), M_DEVBUF, M_NOWAIT | M_ZERO))) {
		printf("%s: Unable to allocate Rx queue memory\n",
		    sc->sc_dev.dv_xname);
		return (ENOMEM);
	}

	/* Allocate Rx descriptor memory */
	for (i = 0, rxq = vsi->rx_queues; i < sc->sc_nqueues; i++, rxq++) {
		rxq->rx_base = ice_alloc_dma_mem(&sc->hw, &rxq->rx_desc_mem,
		    sc->isc_nrxd[i] * sizeof(union ice_32b_rx_flex_desc));
		if (rxq->rx_base == NULL) {
			printf("%s: Unable to allocate Rx descriptor memory\n",
			    sc->sc_dev.dv_xname);
			err = ENOMEM;
			goto free_rx_queues;
		}
		rxq->rx_paddr = rxq->rx_desc_mem.pa;
	}

	/* Create Rx queue DMA maps. */
	for (i = 0, rxq = vsi->rx_queues; i < sc->sc_nqueues; i++, rxq++) {
		struct ice_rx_map *map;
		int j;

		rxq->rx_map = mallocarray(sc->isc_nrxd[i], sizeof(*map),
		    M_DEVBUF, M_NOWAIT| M_ZERO);
		if (rxq->rx_map == NULL) {
			printf("%s: could not allocate Rx DMA map\n",
			    sc->sc_dev.dv_xname);
			err = ENOMEM;
			goto free_rx_queues;
		}

		for (j = 0; j < sc->isc_nrxd[i]; j++) {
			map = &rxq->rx_map[j];
			if (bus_dmamap_create(sc->sc_dmat, hardmtu, 1,
			    hardmtu, 0,
			    BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT,
			    &map->rxm_map) != 0) {
				printf("%s: could not allocate Rx DMA map\n",
				    sc->sc_dev.dv_xname);
				err = ENOMEM;
				goto free_rx_queues;
			}
		}
	}

	/* Assign queues from PF space to the main VSI */
	err = ice_resmgr_assign_contiguous(&sc->rx_qmgr, vsi->rx_qmap,
	    sc->sc_nqueues);
	if (err) {
		printf("%s: Unable to assign PF queues: error %d\n",
		    sc->sc_dev.dv_xname, err);
		goto free_rx_queues;
	}
	vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS;
#if 0
	/* Add Rx queue sysctls context */
	ice_vsi_add_rxqs_ctx(vsi);
#endif
	for (i = 0, rxq = vsi->rx_queues; i < sc->sc_nqueues; i++, rxq++) {
		rxq->me = i;
		rxq->vsi = vsi;

		/* store the queue size for easier access */
		rxq->desc_count = sc->isc_nrxd[i];

		/* set tail address */
		rxq->tail = QRX_TAIL(vsi->rx_qmap[i]);
#if 0
		ice_add_rxq_sysctls(rxq);
#endif
		if_rxr_init(&rxq->rxq_acct, ICE_MIN_DESC_COUNT,
		    rxq->desc_count - 1);
		timeout_set(&rxq->rxq_refill, ice_rxrefill, rxq);

		rxq->rxq_cons = rxq->rxq_prod = 0;
		rxq->rxq_m_head = NULL;
		rxq->rxq_m_tail = &rxq->rxq_m_head;
	}

	vsi->num_rx_queues = sc->sc_nqueues;

	return (0);

free_rx_queues:
	ice_free_rx_queues(sc);
	return err;
}

/**
 * ice_aq_start_stop_dcbx - Start/Stop DCBX service in FW
 * @hw: pointer to the HW struct
 * @start_dcbx_agent: True if DCBX Agent needs to be started
 *		      False if DCBX Agent needs to be stopped
 * @dcbx_agent_status: FW indicates back the DCBX agent status
 *		       True if DCBX Agent is active
 *		       False if DCBX Agent is stopped
 * @cd: pointer to command details structure or NULL
 *
 * Start/Stop the embedded dcbx Agent. In case that this wrapper function
 * returns ICE_SUCCESS, caller will need to check if FW returns back the same
 * value as stated in dcbx_agent_status, and react accordingly. (0x0A09)
 */
enum ice_status
ice_aq_start_stop_dcbx(struct ice_hw *hw, bool start_dcbx_agent,
		       bool *dcbx_agent_status, struct ice_sq_cd *cd)
{
	struct ice_aqc_lldp_stop_start_specific_agent *cmd;
	enum ice_adminq_opc opcode;
	struct ice_aq_desc desc;
	enum ice_status status;

	cmd = &desc.params.lldp_agent_ctrl;

	opcode = ice_aqc_opc_lldp_stop_start_specific_agent;

	ice_fill_dflt_direct_cmd_desc(&desc, opcode);

	if (start_dcbx_agent)
		cmd->command = ICE_AQC_START_STOP_AGENT_START_DCBX;

	status = ice_aq_send_cmd(hw, &desc, NULL, 0, cd);

	*dcbx_agent_status = false;

	if (status == ICE_SUCCESS &&
	    cmd->command == ICE_AQC_START_STOP_AGENT_START_DCBX)
		*dcbx_agent_status = true;

	return status;
}

/**
 * ice_get_dcbx_status
 * @hw: pointer to the HW struct
 *
 * Get the DCBX status from the Firmware
 */
uint8_t
ice_get_dcbx_status(struct ice_hw *hw)
{
	uint32_t reg;

	reg = ICE_READ(hw, PRTDCB_GENS);
	return (uint8_t)((reg & PRTDCB_GENS_DCBX_STATUS_M) >>
		    PRTDCB_GENS_DCBX_STATUS_S);
}

/**
 * ice_start_dcbx_agent - Start DCBX agent in FW via AQ command
 * @sc: the device softc
 *
 * @pre device is DCB capable and the FW LLDP agent has started
 *
 * Checks DCBX status and starts the DCBX agent if it is not in
 * a valid state via an AQ command.
 */
void
ice_start_dcbx_agent(struct ice_softc *sc)
{
	struct ice_hw *hw = &sc->hw;
	bool dcbx_agent_status;
	enum ice_status status;

	hw->port_info->qos_cfg.dcbx_status = ice_get_dcbx_status(hw);

	if (hw->port_info->qos_cfg.dcbx_status != ICE_DCBX_STATUS_DONE &&
	    hw->port_info->qos_cfg.dcbx_status != ICE_DCBX_STATUS_IN_PROGRESS) {
		/*
		 * Start DCBX agent, but not LLDP. The return value isn't
		 * checked here because a more detailed dcbx agent status is
		 * retrieved and checked in ice_init_dcb() and elsewhere.
		 */
		status = ice_aq_start_stop_dcbx(hw, true, &dcbx_agent_status,
		    NULL);
		if (status && hw->adminq.sq_last_status != ICE_AQ_RC_EPERM)
			printf("%s: start_stop_dcbx failed, err %s aq_err %s\n",
			    sc->sc_dev.dv_xname, ice_status_str(status),
			    ice_aq_str(hw->adminq.sq_last_status));
	}
}

/**
 * ice_aq_cfg_lldp_mib_change
 * @hw: pointer to the HW struct
 * @ena_update: Enable or Disable event posting
 * @cd: pointer to command details structure or NULL
 *
 * Enable or Disable posting of an event on ARQ when LLDP MIB
 * associated with the interface changes (0x0A01)
 */
enum ice_status
ice_aq_cfg_lldp_mib_change(struct ice_hw *hw, bool ena_update,
			   struct ice_sq_cd *cd)
{
	struct ice_aqc_lldp_set_mib_change *cmd;
	struct ice_aq_desc desc;

	cmd = &desc.params.lldp_set_event;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_lldp_set_mib_change);

	if (!ena_update)
		cmd->command |= ICE_AQ_LLDP_MIB_UPDATE_DIS;
	else
		cmd->command |= ICE_AQ_LLDP_MIB_PENDING_ENABLE <<
				ICE_AQ_LLDP_MIB_PENDING_S;

	return ice_aq_send_cmd(hw, &desc, NULL, 0, cd);
}

/**
 * ice_init_dcb
 * @hw: pointer to the HW struct
 * @enable_mib_change: enable MIB change event
 *
 * Update DCB configuration from the Firmware
 */
enum ice_status
ice_init_dcb(struct ice_hw *hw, bool enable_mib_change)
{
	struct ice_qos_cfg *qos_cfg = &hw->port_info->qos_cfg;
	enum ice_status ret = ICE_SUCCESS;

	if (!hw->func_caps.common_cap.dcb)
		return ICE_ERR_NOT_SUPPORTED;

	qos_cfg->is_sw_lldp = true;

	/* Get DCBX status */
	qos_cfg->dcbx_status = ice_get_dcbx_status(hw);

	if (qos_cfg->dcbx_status == ICE_DCBX_STATUS_DONE ||
	    qos_cfg->dcbx_status == ICE_DCBX_STATUS_IN_PROGRESS ||
	    qos_cfg->dcbx_status == ICE_DCBX_STATUS_NOT_STARTED) {
		/* Get current DCBX configuration */
		ret = ice_get_dcb_cfg(hw->port_info);
		if (ret)
			return ret;
		qos_cfg->is_sw_lldp = false;
	} else if (qos_cfg->dcbx_status == ICE_DCBX_STATUS_DIS) {
		return ICE_ERR_NOT_READY;
	}

	/* Configure the LLDP MIB change event */
	if (enable_mib_change) {
		ret = ice_aq_cfg_lldp_mib_change(hw, true, NULL);
		if (ret)
			qos_cfg->is_sw_lldp = true;
	}

	return ret;
}

/**
 * ice_aq_query_pfc_mode - Query PFC mode
 * @hw: pointer to the HW struct
 * @pfcmode_ret: Return PFC mode
 * @cd: pointer to command details structure or NULL
 *
 * This will return an indication if DSCP-based PFC or VLAN-based PFC
 * is enabled. (0x0302)
 */
enum ice_status
ice_aq_query_pfc_mode(struct ice_hw *hw, uint8_t *pfcmode_ret,
    struct ice_sq_cd *cd)
{
	struct ice_aqc_set_query_pfc_mode *cmd;
	struct ice_aq_desc desc;
	enum ice_status status;

	cmd = &desc.params.set_query_pfc_mode;

	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_query_pfc_mode);

	status = ice_aq_send_cmd(hw, &desc, NULL, 0, cd);

	if (!status)
		*pfcmode_ret = cmd->pfc_mode;

	return status;
}

/**
 * ice_init_dcb_setup - Initialize DCB settings for HW
 * @sc: the device softc
 *
 * This needs to be called after the fw_lldp_agent sysctl is added, since that
 * can update the device's LLDP agent status if a tunable value is set.
 *
 * Get and store the initial state of DCB settings on driver load. Print out
 * informational messages as well.
 */
void
ice_init_dcb_setup(struct ice_softc *sc)
{
	struct ice_dcbx_cfg *local_dcbx_cfg;
	struct ice_hw *hw = &sc->hw;
	enum ice_status status;
	uint8_t pfcmode_ret;

	/* Don't do anything if DCB isn't supported */
	if (!ice_is_bit_set(sc->feat_cap, ICE_FEATURE_DCB)) {
		DPRINTF("%s: No DCB support\n", __func__);
		return;
	}

	/* Starts DCBX agent if it needs starting */
	ice_start_dcbx_agent(sc);

	/* This sets hw->port_info->qos_cfg.is_sw_lldp */
	status = ice_init_dcb(hw, true);

	/* If there is an error, then FW LLDP is not in a usable state */
	if (status != 0 && status != ICE_ERR_NOT_READY) {
		/* Don't print an error message if the return code from the AQ
		 * cmd performed in ice_init_dcb() is EPERM; that means the
		 * FW LLDP engine is disabled, and that is a valid state.
		 */
		if (!(status == ICE_ERR_AQ_ERROR &&
		      hw->adminq.sq_last_status == ICE_AQ_RC_EPERM)) {
			printf("%s: DCB init failed, err %s aq_err %s\n",
			    sc->sc_dev.dv_xname, ice_status_str(status),
			    ice_aq_str(hw->adminq.sq_last_status));
		}
		hw->port_info->qos_cfg.dcbx_status = ICE_DCBX_STATUS_NOT_STARTED;
	}

	switch (hw->port_info->qos_cfg.dcbx_status) {
	case ICE_DCBX_STATUS_DIS:
		DNPRINTF(ICE_DBG_DCB, "%s: DCBX disabled\n", __func__);
		break;
	case ICE_DCBX_STATUS_NOT_STARTED:
		DNPRINTF(ICE_DBG_DCB, "%s: DCBX not started\n", __func__);
		break;
	case ICE_DCBX_STATUS_MULTIPLE_PEERS:
		DNPRINTF(ICE_DBG_DCB, "%s: DCBX detected multiple peers\n",
		    __func__);
		break;
	default:
		break;
	}

	/* LLDP disabled in FW */
	if (hw->port_info->qos_cfg.is_sw_lldp) {
		ice_add_rx_lldp_filter(sc);
		DPRINTF("%s: Firmware LLDP agent disabled\n", __func__);
	}

	/* Query and cache PFC mode */
	status = ice_aq_query_pfc_mode(hw, &pfcmode_ret, NULL);
	if (status) {
		printf("%s: PFC mode query failed, err %s aq_err %s\n",
		    sc->sc_dev.dv_xname, ice_status_str(status),
		    ice_aq_str(hw->adminq.sq_last_status));
	}
	local_dcbx_cfg = &hw->port_info->qos_cfg.local_dcbx_cfg;
	switch (pfcmode_ret) {
	case ICE_AQC_PFC_VLAN_BASED_PFC:
		local_dcbx_cfg->pfc_mode = ICE_QOS_MODE_VLAN;
		break;
	case ICE_AQC_PFC_DSCP_BASED_PFC:
		local_dcbx_cfg->pfc_mode = ICE_QOS_MODE_DSCP;
		break;
	default:
		/* DCB is disabled, but we shouldn't get here */
		break;
	}

	/* Set default SW MIB for init */
	ice_set_default_local_mib_settings(sc);

	ice_set_bit(ICE_FEATURE_DCB, sc->feat_en);
}

/**
 * ice_set_link_management_mode -- Strict or lenient link management
 * @sc: device private structure
 *
 * Some NVMs give the adapter the option to advertise a superset of link
 * configurations.  This checks to see if that option is enabled.
 * Further, the NVM could also provide a specific set of configurations
 * to try; these are cached in the driver's private structure if they
 * are available.
 */
void
ice_set_link_management_mode(struct ice_softc *sc)
{
	struct ice_port_info *pi = sc->hw.port_info;
	struct ice_link_default_override_tlv tlv = { 0 };
	enum ice_status status;

	/* Port must be in strict mode if FW version is below a certain
	 * version. (i.e. Don't set lenient mode features)
	 */
	if (!(ice_fw_supports_link_override(&sc->hw)))
		return;

	status = ice_get_link_default_override(&tlv, pi);
	if (status != ICE_SUCCESS) {
		DPRINTF("%s: ice_get_link_default_override failed; "
		    "status %s, aq_err %s\n", __func__, ice_status_str(status),
		    ice_aq_str(sc->hw.adminq.sq_last_status));
		return;
	}
#if 0
	if (sc->hw.debug_mask & ICE_DBG_LINK)
		ice_print_ldo_tlv(sc, &tlv);
#endif
	/* Set lenient link mode */
	if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_LENIENT_LINK_MODE) &&
	    (!(tlv.options & ICE_LINK_OVERRIDE_STRICT_MODE)))
		ice_set_bit(ICE_FEATURE_LENIENT_LINK_MODE, sc->feat_en);

	/* FW supports reporting a default configuration */
	if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_LINK_MGMT_VER_2) &&
	    ice_fw_supports_report_dflt_cfg(&sc->hw)) {
		ice_set_bit(ICE_FEATURE_LINK_MGMT_VER_2, sc->feat_en);
		/* Knowing we're at a high enough firmware revision to
		 * support this link management configuration, we don't
		 * need to check/support earlier versions.
		 */
		return;
	}

	/* Default overrides only work if in lenient link mode */
	if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_LINK_MGMT_VER_1) &&
	    ice_is_bit_set(sc->feat_en, ICE_FEATURE_LENIENT_LINK_MODE) &&
	    (tlv.options & ICE_LINK_OVERRIDE_EN))
		ice_set_bit(ICE_FEATURE_LINK_MGMT_VER_1, sc->feat_en);

	/* Cache the LDO TLV structure in the driver, since it
	 * won't change during the driver's lifetime.
	 */
	sc->ldo_tlv = tlv;
}

/**
 * ice_init_saved_phy_cfg -- Set cached user PHY cfg settings with NVM defaults
 * @sc: device private structure
 *
 * This should be called before the tunables for these link settings
 * (e.g. advertise_speed) are added -- so that these defaults don't overwrite
 * the cached values that the sysctl handlers will write.
 *
 * This also needs to be called before ice_init_link_configuration, to ensure
 * that there are sane values that can be written if there is media available
 * in the port.
 */
void
ice_init_saved_phy_cfg(struct ice_softc *sc)
{
	struct ice_port_info *pi = sc->hw.port_info;
	struct ice_aqc_get_phy_caps_data pcaps = { 0 };
	enum ice_status status;
	uint64_t phy_low, phy_high;
	uint8_t report_mode = ICE_AQC_REPORT_TOPO_CAP_MEDIA;

	if (ice_is_bit_set(sc->feat_en, ICE_FEATURE_LINK_MGMT_VER_2))
		report_mode = ICE_AQC_REPORT_DFLT_CFG;
	status = ice_aq_get_phy_caps(pi, false, report_mode, &pcaps, NULL);
	if (status != ICE_SUCCESS) {
		DPRINTF("%s: ice_aq_get_phy_caps (%s) failed; status %s, "
		    "aq_err %s\n", __func__,
		    report_mode == ICE_AQC_REPORT_DFLT_CFG ? "DFLT" : "w/MEDIA",
		    ice_status_str(status),
		    ice_aq_str(sc->hw.adminq.sq_last_status));
		return;
	}

	phy_low = le64toh(pcaps.phy_type_low);
	phy_high = le64toh(pcaps.phy_type_high);

	/* Save off initial config parameters */
	pi->phy.curr_user_speed_req =
	   ice_aq_phy_types_to_link_speeds(phy_low, phy_high);
	pi->phy.curr_user_fec_req = ice_caps_to_fec_mode(pcaps.caps,
	    pcaps.link_fec_options);
	pi->phy.curr_user_fc_req = ice_caps_to_fc_mode(pcaps.caps);
}

/**
 * ice_read_pba_string - Reads part number string from NVM
 * @hw: pointer to hardware structure
 * @pba_num: stores the part number string from the NVM
 * @pba_num_size: part number string buffer length
 *
 * Reads the part number string from the NVM.
 */
enum ice_status
ice_read_pba_string(struct ice_hw *hw, uint8_t *pba_num, uint32_t pba_num_size)
{
	uint16_t pba_tlv, pba_tlv_len;
	enum ice_status status;
	uint16_t pba_word, pba_size;
	uint16_t i;

	status = ice_get_pfa_module_tlv(hw, &pba_tlv, &pba_tlv_len,
					ICE_SR_PBA_BLOCK_PTR);
	if (status != ICE_SUCCESS) {
		DNPRINTF(ICE_DBG_INIT, "%s: Failed to read PBA Block TLV.\n",
		    __func__);
		return status;
	}

	/* pba_size is the next word */
	status = ice_read_sr_word(hw, (pba_tlv + 2), &pba_size);
	if (status != ICE_SUCCESS) {
		DNPRINTF(ICE_DBG_INIT, "%s: Failed to read PBA Section size.\n",
		    __func__);
		return status;
	}

	if (pba_tlv_len < pba_size) {
		DNPRINTF(ICE_DBG_INIT, "%s: Invalid PBA Block TLV size.\n",
		    __func__);
		return ICE_ERR_INVAL_SIZE;
	}

	/* Subtract one to get PBA word count (PBA Size word is included in
	 * total size)
	 */
	pba_size--;
	if (pba_num_size < (((uint32_t)pba_size * 2) + 1)) {
		DNPRINTF(ICE_DBG_INIT, "%s: Buffer too small for PBA data.\n",
		    __func__);
		return ICE_ERR_PARAM;
	}

	for (i = 0; i < pba_size; i++) {
		status = ice_read_sr_word(hw, (pba_tlv + 2 + 1) + i, &pba_word);
		if (status != ICE_SUCCESS) {
			DNPRINTF(ICE_DBG_INIT,
			    "%s: Failed to read PBA Block word %d.\n",
			    __func__, i);
			return status;
		}

		pba_num[(i * 2)] = (pba_word >> 8) & 0xFF;
		pba_num[(i * 2) + 1] = pba_word & 0xFF;
	}
	pba_num[(pba_size * 2)] = '\0';

	return status;
}

/**
 * ice_cfg_pba_num - Determine if PBA Number is retrievable
 * @sc: the device private softc structure
 *
 * Sets the feature flag for the existence of a PBA number
 * based on the success of the read command.  This does not
 * cache the result.
 */
void
ice_cfg_pba_num(struct ice_softc *sc)
{
	uint8_t pba_string[32] = "";

	if ((ice_is_bit_set(sc->feat_cap, ICE_FEATURE_HAS_PBA)) &&
	    (ice_read_pba_string(&sc->hw, pba_string, sizeof(pba_string)) == 0))
		ice_set_bit(ICE_FEATURE_HAS_PBA, sc->feat_en);
}

/**
 * ice_init_link_configuration -- Setup link in different ways depending
 * on whether media is available or not.
 * @sc: device private structure
 *
 * Called at the end of the attach process to either set default link
 * parameters if there is media available, or force HW link down and
 * set a state bit if there is no media.
 */
void
ice_init_link_configuration(struct ice_softc *sc)
{
	struct ice_port_info *pi = sc->hw.port_info;
	struct ice_hw *hw = &sc->hw;
	enum ice_status status;

	pi->phy.get_link_info = true;
	status = ice_get_link_status(pi, &sc->link_up);
	if (status != ICE_SUCCESS) {
		DPRINTF("%s: ice_get_link_status failed; status %s, "
		    "aq_err %s\n", __func__, ice_status_str(status),
		    ice_aq_str(hw->adminq.sq_last_status));
		return;
	}

	if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) {
		ice_clear_state(&sc->state, ICE_STATE_NO_MEDIA);
		/* Apply default link settings */
		if (!ice_test_state(&sc->state,
		    ICE_STATE_LINK_ACTIVE_ON_DOWN)) {
			ice_set_link(sc, false);
			ice_set_state(&sc->state,
			    ICE_STATE_LINK_STATUS_REPORTED);
		} else
			ice_apply_saved_phy_cfg(sc, ICE_APPLY_LS_FEC_FC);
	} else {
		 /*
		  * Set link down, and poll for media available in timer.
		  * This prevents the driver from receiving spurious
		  * link-related events.
		  */
		ice_set_state(&sc->state, ICE_STATE_NO_MEDIA);
		status = ice_aq_set_link_restart_an(pi, false, NULL);
		if (status != ICE_SUCCESS &&
		    hw->adminq.sq_last_status != ICE_AQ_RC_EMODE) {
			DPRINTF("%s: ice_aq_set_link_restart_an: status %s, "
			    "aq_err %s\n", __func__, ice_status_str(status),
			    ice_aq_str(hw->adminq.sq_last_status));
		}
	}
}

void
ice_attach_hook(struct device *self)
{
	struct ice_softc *sc = (void *)self;
	struct ice_hw *hw = &sc->hw;
	struct ifnet *ifp = &sc->sc_ac.ac_if;
	enum ice_status status;
	unsigned nmsix, nqueues_max, nqueues;
	int err;

	KASSERT(!cold);

	/*
	 * Attempt to load a firmware package.
	 * Success indicates a change was made that requires a reinitialization
	 * of the hardware
	 */
	status = ice_load_pkg_file(sc);
	if (status == ICE_SUCCESS) {
		ice_reinit_hw(sc);
		return;
	}

	err = ice_init_link_events(sc);
	if (err)
		goto deinit_hw;

	/* Initialize VLAN mode in FW; if dual VLAN mode is supported by the package
	 * and firmware, this will force them to use single VLAN mode.
	 */
	status = ice_set_vlan_mode(hw);
	if (status) {
		err = EIO;
		DPRINTF("%s: Unable to initialize VLAN mode, "
		    "err %s aq_err %s\n", __func__,
		    ice_status_str(status),
		    ice_aq_str(hw->adminq.sq_last_status));
		goto deinit_hw;
	}

	ice_print_nvm_version(sc);

	/* Setup the MAC address */
	err = if_setlladdr(ifp, hw->port_info->mac.perm_addr);
	if (err)
		printf("%s: could not set MAC address (error %d)\n",
		    sc->sc_dev.dv_xname, err);

	ice_setup_scctx(sc);
	if (ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE))
		ice_set_safe_mode_caps(hw);

	/*
	 * Figure out how many queues we can use.
	 * Require at least two MSIX vectors, one for traffic and
	 * one for misc causes.
	 */
	nmsix = MIN(sc->sc_nmsix_max,
	    hw->func_caps.common_cap.num_msix_vectors);
	if (nmsix < 2) {
		printf("%s: insufficient amount of MSIx vectors available\n",
		    sc->sc_dev.dv_xname);
		goto deinit_hw;
	}
	sc->sc_nmsix = nmsix;
	nqueues_max = MIN(hw->func_caps.common_cap.num_rxq,
	    hw->func_caps.common_cap.num_txq);
	sc->sc_intrmap = intrmap_create(&sc->sc_dev, sc->sc_nmsix - 1,
	    nqueues_max, INTRMAP_POWEROF2);
	nqueues = intrmap_count(sc->sc_intrmap);
	KASSERT(nqueues > 0);
	KASSERT(powerof2(nqueues));
	sc->sc_nqueues = MIN(nqueues, sc->sc_nvectors);
	DPRINTF("%s: %d MSIx vector%s available, using %d queue%s\n", __func__,
	    sc->sc_nmsix, sc->sc_nmsix > 1 ? "s" : "",
	    sc->sc_nqueues, sc->sc_nqueues > 1 ? "s" : "");

	/* Initialize the Tx queue manager */
	err = ice_resmgr_init(&sc->tx_qmgr, hw->func_caps.common_cap.num_txq);
	if (err) {
		printf("%s: Unable to initialize Tx queue manager: err %d\n",
		    sc->sc_dev.dv_xname, err);
		goto deinit_hw;
	}

	/* Initialize the Rx queue manager */
	err = ice_resmgr_init(&sc->rx_qmgr, hw->func_caps.common_cap.num_rxq);
	if (err) {
		printf("%s: Unable to initialize Rx queue manager: %d\n",
		    sc->sc_dev.dv_xname, err);
		goto free_tx_qmgr;
	}

	/* Initialize the PF device interrupt resource manager */
	err = ice_alloc_intr_tracking(sc);
	if (err)
		/* Errors are already printed */
		goto free_rx_qmgr;

	/* Determine maximum number of VSIs we'll prepare for */
	sc->num_available_vsi = MIN(ICE_MAX_VSI_AVAILABLE,
				    hw->func_caps.guar_num_vsi);
	if (!sc->num_available_vsi) {
		err = EIO;
		printf("%s: No VSIs allocated to host\n",
		    sc->sc_dev.dv_xname);
		goto free_intr_tracking;
	}

	/* Allocate storage for the VSI pointers */
	sc->all_vsi = (struct ice_vsi **)
		mallocarray(sc->num_available_vsi, sizeof(struct ice_vsi *),
		       M_DEVBUF, M_WAITOK | M_ZERO);
	if (sc->all_vsi == NULL) {
		err = ENOMEM;
		printf("%s: Unable to allocate VSI array\n",
		    sc->sc_dev.dv_xname);
		goto free_intr_tracking;
	}

	/*
	 * Prepare the statically allocated primary PF VSI in the softc
	 * structure. Other VSIs will be dynamically allocated as needed.
	 */
	ice_setup_pf_vsi(sc);

	err = ice_alloc_vsi_qmap(&sc->pf_vsi, sc->isc_ntxqsets_max,
	    sc->isc_nrxqsets_max);
	if (err) {
		printf("%s: Unable to allocate VSI Queue maps\n",
		    sc->sc_dev.dv_xname);
		goto free_main_vsi;
	}

	/* Allocate MSI-X vectors. */
	err = ice_allocate_msix(sc);
	if (err)
		goto free_main_vsi;

	err = ice_tx_queues_alloc(sc);
	if (err)
		goto free_main_vsi;

	err = ice_rx_queues_alloc(sc);
	if (err)
		goto free_queues;

	err = ice_initialize_vsi(&sc->pf_vsi);
	if (err)
		goto free_queues;

	if_attach_queues(ifp, sc->sc_nqueues);
	if_attach_iqueues(ifp, sc->sc_nqueues);

	/* Enable FW health event reporting */
	ice_init_health_events(sc);
#if 0
	/* Configure the main PF VSI for RSS */
	err = ice_config_rss(&sc->pf_vsi);
	if (err) {
		device_printf(sc->dev,
			      "Unable to configure RSS for the main VSI, err %s\n",
			      ice_err_str(err));
		return err;
	}

	/* Configure switch to drop transmitted LLDP and PAUSE frames */
	err = ice_cfg_pf_ethertype_filters(sc);
	if (err)
		return err;

	ice_get_and_print_bus_info(sc);
#endif
	ice_set_link_management_mode(sc);

	ice_init_saved_phy_cfg(sc);

	ice_cfg_pba_num(sc);
#if 0
	/* Set a default value for PFC mode on attach since the FW state is unknown
	 * before sysctl tunables are executed and it can't be queried. This fixes an
	 * issue when loading the driver with the FW LLDP agent enabled but the FW
	 * was previously in DSCP PFC mode.
	 */
	status = ice_aq_set_pfc_mode(&sc->hw, ICE_AQC_PFC_VLAN_BASED_PFC, NULL);
	if (status != ICE_SUCCESS)
		device_printf(sc->dev, "Setting pfc mode failed, status %s\n", ice_status_str(status));

	ice_add_device_sysctls(sc);
#endif
	/* Get DCBX/LLDP state and start DCBX agent */
	ice_init_dcb_setup(sc);

	/* Setup link configuration parameters */
	ice_init_link_configuration(sc);
	ice_update_link_status(sc, true);

	/* Configure interrupt causes for the administrative interrupt */
	ice_configure_misc_interrupts(sc);

	/* Enable ITR 0 right away, so that we can handle admin interrupts */
	ice_enable_intr(&sc->hw, 0);
#if 0
	err = ice_rdma_pf_attach(sc);
	if (err)
		return (err);
#endif
	if (ice_test_state(&sc->state, ICE_STATE_LINK_ACTIVE_ON_DOWN) &&
		 !ice_test_state(&sc->state, ICE_STATE_NO_MEDIA))
		ice_set_state(&sc->state, ICE_STATE_FIRST_INIT_LINK);

	ice_clear_state(&sc->state, ICE_STATE_ATTACHING);
	return;

free_queues:
	ice_free_tx_queues(sc);	
	ice_free_rx_queues(sc);	
free_main_vsi:
	/* ice_release_vsi will free the queue maps if they were allocated */
	ice_release_vsi(&sc->pf_vsi);
	free(sc->all_vsi, M_DEVBUF,
	    sc->num_available_vsi * sizeof(struct ice_vis *));
	sc->all_vsi = NULL;
free_intr_tracking:
	ice_free_intr_tracking(sc);
free_rx_qmgr:
	ice_resmgr_destroy(&sc->rx_qmgr);
free_tx_qmgr:
	ice_resmgr_destroy(&sc->tx_qmgr);
deinit_hw:
	ice_deinit_hw(hw);
}

void
ice_attach(struct device *parent, struct device *self, void *aux)
{
	struct ice_softc *sc = (void *)self;
	struct ifnet *ifp = &sc->sc_ac.ac_if;
	struct ice_hw *hw = &sc->hw;
	struct pci_attach_args *pa = aux;
	enum ice_fw_modes fw_mode;
	pcireg_t memtype;
	enum ice_status status;
	int err, i;

	ice_set_state(&sc->state, ICE_STATE_ATTACHING);

	sc->sc_pid = PCI_PRODUCT(pa->pa_id);
	sc->sc_pct = pa->pa_pc;
	sc->sc_pcitag = pa->pa_tag;
	sc->sc_dmat = pa->pa_dmat;

	hw->hw_sc = sc;

	task_set(&sc->sc_admin_task, ice_if_update_admin_status, sc);
	timeout_set(&sc->sc_admin_timer, ice_admin_timer, sc);

	memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_MAPREG_START);
	err = pci_mapreg_map(pa, PCI_MAPREG_START, memtype, 0,
	    &sc->sc_st, &sc->sc_sh, NULL, &sc->sc_sz, 0);
	if (err) {
		printf("%s: can't map mem space\n", sc->sc_dev.dv_xname);
		return;
	}

	ice_set_ctrlq_len(hw);

	fw_mode = ice_get_fw_mode(hw);
	if (fw_mode == ICE_FW_MODE_REC) {
		printf("%s: firmware is in recovery mode\n",
		    sc->sc_dev.dv_xname);
#if 0
		err = ice_attach_pre_recovery_mode(sc);
		if (err)
			goto free_pci_mapping;
#endif
		return;
	}

	/* Initialize the hw data structure */
	status = ice_init_hw(hw);
	if (status) {
		if (status == ICE_ERR_FW_API_VER) {
			printf("%s: incompatible firmware API version\n",
			    sc->sc_dev.dv_xname);
#if 0
			/* Enter recovery mode, so that the driver remains
			 * loaded. This way, if the system administrator
			 * cannot update the driver, they may still attempt to
			 * downgrade the NVM.
			 */
			err = ice_attach_pre_recovery_mode(sc);
			if (err)
				goto free_pci_mapping;
#endif
			return;
		} else {
			printf("%s: could not initialize hardware, "
			    "status %s aq_err %s\n",
			    sc->sc_dev.dv_xname, ice_status_str(status),
			    ice_aq_str(hw->adminq.sq_last_status));
			return;
		}
	}

	if (pci_intr_map_msix(pa, 0, &sc->sc_ih) == 0) {
		unsigned int nmsix = pci_intr_msix_count(pa);

		/*
		 * Require at least two MSIX vectors, one for traffic and
		 * one for misc causes.
		 */
		if (nmsix < 2) {
			printf(": can't map interrupt\n");
			return;
		}
		sc->sc_nmsix_max = nmsix;
	} else {
		printf(": can't map interrupt\n");
		return;
	}

	/*
	 * Map an extra MSI-X vector per CPU, up to a hard-coded limit.
	 * We may not need them all but we can only figure out the supported
	 * number of queues once firmware is loaded.
	 */
	sc->sc_nvectors = MIN(sc->sc_nmsix_max, ncpus);
	sc->sc_nvectors = MIN(sc->sc_nvectors, ICE_MAX_VECTORS);
	sc->sc_vectors = mallocarray(sizeof(*sc->sc_vectors), sc->sc_nvectors,
	    M_DEVBUF, M_WAITOK|M_CANFAIL|M_ZERO);
	if (sc->sc_vectors == NULL) {
		printf(": unable to allocate MSIx interrupt vector array\n");
		return;
	}
	for (i = 0; i < sc->sc_nvectors; i++) {
		struct ice_intr_vector *iv = &sc->sc_vectors[i];
		int v = i + 1; /* 0 is used for adminq */

		iv->iv_sc = sc;
		iv->iv_qid = i;
		snprintf(iv->iv_name, sizeof(iv->iv_name),
		    "%s:%u", sc->sc_dev.dv_xname, i);
		if (pci_intr_map_msix(pa, v, &iv->ih)) {
			printf(": unable to map MSI-X vector %d\n", v);
			goto free_vectors;
		}
	}
	
	printf(": %s\n", pci_intr_string(sc->sc_pct, sc->sc_ih));

	ice_init_device_features(sc);

	/* Keep flag set by default */
	ice_set_state(&sc->state, ICE_STATE_LINK_ACTIVE_ON_DOWN);

	/* Notify firmware of the device driver version */
	err = ice_send_version(sc);
	if (err)
		goto deinit_hw;

	ifp->if_softc = sc;
	strlcpy(ifp->if_xname, sc->sc_dev.dv_xname, IFNAMSIZ);
	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
	ifp->if_xflags = IFXF_MPSAFE;
	ifp->if_ioctl = ice_ioctl;
	ifp->if_qstart = ice_start;
	ifp->if_watchdog = ice_watchdog;
	ifp->if_hardmtu = ice_hardmtu(hw);

	ifq_init_maxlen(&ifp->if_snd, ICE_DEFAULT_DESC_COUNT);

	/* Initialize ifmedia structures. */
	ifmedia_init(&sc->media, IFM_IMASK, ice_media_change, ice_media_status);

	ice_add_media_types(sc, &sc->media);

	if_attach(ifp);
	ether_ifattach(ifp);

	config_mountroot(self, ice_attach_hook);
	return;

deinit_hw:
	ice_deinit_hw(hw);
free_vectors:
	free(sc->sc_vectors, M_DEVBUF,
	    sc->sc_nvectors * sizeof(*sc->sc_vectors));
	sc->sc_vectors = NULL;
	sc->sc_nvectors = 0;
}

struct cfdriver ice_cd = {
	NULL, "ice", DV_IFNET
};

const struct cfattach ice_ca = {
	sizeof(struct ice_softc), ice_match, ice_attach,
	NULL, NULL,
};