path: root/sys/dev/acpi/acpidmar.h

/*
 * Copyright (c) 2015 Jordan Hargrave <jordan_hargrave@hotmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#ifndef _DEV_ACPI_DMARREG_H_
#define _DEV_ACPI_DMARREG_H_

/*#define IOMMU_DEBUG*/

#define VTD_STRIDE_MASK 0x1FF
#define VTD_STRIDE_SIZE 9
#define VTD_PAGE_SIZE   4096
#define VTD_PAGE_MASK   0xFFF
#define VTD_PTE_MASK    0x0000FFFFFFFFF000LL

#define VTD_LEVEL0	12
#define VTD_LEVEL1	21
#define VTD_LEVEL2	30 /* Minimum level supported */
#define VTD_LEVEL3	39 /* Also supported */
#define VTD_LEVEL4	48
#define VTD_LEVEL5	57

#define _xbit(x,y) (((x)>> (y)) & 1)
#define _xfld(x,y) (uint32_t)(((x)>> y##_SHIFT) & y##_MASK)

#define VTD_AWTOLEVEL(x)    (((x) - 30) / VTD_STRIDE_SIZE)
#define VTD_LEVELTOAW(x)    (((x) * VTD_STRIDE_SIZE) + 30)

#define DMAR_VER_REG		0x00    /* 32:Arch version supported by this IOMMU */
#define DMAR_RTADDR_REG		0x20    /* 64:Root entry table */
#define DMAR_FEDATA_REG		0x3c    /* 32:Fault event interrupt data register */
#define DMAR_FEADDR_REG		0x40    /* 32:Fault event interrupt addr register */
#define DMAR_FEUADDR_REG	0x44    /* 32:Upper address register */
#define DMAR_AFLOG_REG		0x58    /* 64:Advanced Fault control */
#define DMAR_PMEN_REG		0x64    /* 32:Enable Protected Memory Region */
#define DMAR_PLMBASE_REG	0x68    /* 32:PMRR Low addr */
#define DMAR_PLMLIMIT_REG	0x6c    /* 32:PMRR low limit */
#define DMAR_PHMBASE_REG	0x70    /* 64:pmrr high base addr */
#define DMAR_PHMLIMIT_REG	0x78    /* 64:pmrr high limit */
#define DMAR_ICS_REG		0x9C    /* 32:Invalidation complete status register */
#define DMAR_IECTL_REG		0xa0    /* 32:Invalidation event control register */
#define DMAR_IEDATA_REG		0xa4    /* 32:Invalidation event data register */
#define DMAR_IEADDR_REG		0xa8    /* 32:Invalidation event address register */
#define DMAR_IEUADDR_REG	0xac    /* 32:Invalidation event upper address register */
#define DMAR_IRTA_REG		0xb8    /* 64:Interrupt remapping table addr register */
#define DMAR_CAP_REG		0x08    /* 64:Hardware supported capabilities */
#define   CAP_PI		(1LL << 59)
#define   CAP_FL1GP		(1LL << 56)
#define   CAP_DRD		(1LL << 55)
#define   CAP_DWD		(1LL << 54)
#define   CAP_MAMV_MASK		0x3F
#define   CAP_MAMV_SHIFT	48LL
#define   cap_mamv(x)		_xfld(x,CAP_MAMV)
#define   CAP_NFR_MASK		0xFF
#define   CAP_NFR_SHIFT		40LL
#define   cap_nfr(x)		(_xfld(x,CAP_NFR) + 1)
#define   CAP_PSI		(1LL << 39)
#define   CAP_SLLPS_MASK	0xF
#define   CAP_SLLPS_SHIFT	34LL
#define   cap_sllps(x)		_xfld(x,CAP_SLLPS)
#define   CAP_FRO_MASK		0x3FF
#define   CAP_FRO_SHIFT		24LL
#define   cap_fro(x)		(_xfld(x,CAP_FRO) * 16)
#define   CAP_ZLR		(1LL << 22)
#define   CAP_MGAW_MASK		0x3F
#define   CAP_MGAW_SHIFT	16LL
#define   cap_mgaw(x)		(_xfld(x,CAP_MGAW) + 1)
#define   CAP_SAGAW_MASK	0x1F
#define   CAP_SAGAW_SHIFT	8LL
#define   cap_sagaw(x)		_xfld(x,CAP_SAGAW)
#define   CAP_CM		(1LL << 7)
#define   CAP_PHMR		(1LL << 6)
#define   CAP_PLMR		(1LL << 5)
#define   CAP_RWBF		(1LL << 4)
#define   CAP_AFL		(1LL << 3)
#define   CAP_ND_MASK		0x7
#define   CAP_ND_SHIFT		0x00
#define   cap_nd(x)		(16 << (((x) & CAP_ND_MASK) << 1))

#define DMAR_ECAP_REG		0x10	/* 64:Extended capabilities supported */
#define   ECAP_PSS_MASK		0x1F
#define   ECAP_PSS_SHIFT	35
#define   ECAP_EAFS		(1LL << 34)
#define   ECAP_NWFS		(1LL << 33)
#define   ECAP_SRS		(1LL << 31)
#define   ECAP_ERS		(1LL << 30)
#define   ECAP_PRS		(1LL << 29)
#define   ECAP_PASID		(1LL << 28)
#define   ECAP_DIS		(1LL << 27)
#define   ECAP_NEST		(1LL << 26)
#define   ECAP_MTS		(1LL << 25)
#define   ECAP_ECS		(1LL << 24)
#define   ECAP_MHMV_MASK	0xF
#define   ECAP_MHMV_SHIFT	0x20
#define   ecap_mhmv(x)		_xfld(x,ECAP_MHMV)
#define   ECAP_IRO_MASK		0x3FF	/* IOTLB Register */
#define   ECAP_IRO_SHIFT	0x8
#define   ecap_iro(x)		(_xfld(x,ECAP_IRO) * 16)
#define   ECAP_SC		(1LL << 7)	/* Snoop Control */
#define   ECAP_PT		(1LL << 6)	/* HW Passthru */
#define   ECAP_EIM		(1LL << 4)
#define   ECAP_IR		(1LL << 3)	/* Interrupt remap */
#define   ECAP_DT		(1LL << 2)	/* Device IOTLB */
#define   ECAP_QI		(1LL << 1)	/* Queued Invalidation */
#define   ECAP_C		(1LL << 0)	/* Coherent cache */

#define DMAR_GCMD_REG		0x18		/* 32:Global command register */
#define   GCMD_TE		(1LL << 31)
#define   GCMD_SRTP		(1LL << 30)
#define   GCMD_SFL		(1LL << 29)
#define   GCMD_EAFL		(1LL << 28)
#define   GCMD_WBF		(1LL << 27)
#define   GCMD_QIE		(1LL << 26)
#define   GCMD_IRE		(1LL << 25)
#define   GCMD_SIRTP		(1LL << 24)
#define   GCMD_CFI		(1LL << 23)

#define DMAR_GSTS_REG		0x1c		/* 32:Global status register */
#define   GSTS_TES		(1LL << 31)
#define   GSTS_RTPS		(1LL << 30)
#define   GSTS_FLS		(1LL << 29)
#define   GSTS_AFLS		(1LL << 28)
#define   GSTS_WBFS		(1LL << 27)
#define   GSTS_QIES		(1LL << 26)
#define   GSTS_IRES		(1LL << 25)
#define   GSTS_IRTPS		(1LL << 24)
#define   GSTS_CFIS		(1LL << 23)

#define DMAR_CCMD_REG		0x28		/* 64:Context command reg */
#define   CCMD_ICC		(1LL << 63)
#define   CCMD_CIRG_MASK	0x3
#define   CCMD_CIRG_SHIFT	61
#define   CCMD_CIRG(x)		((uint64_t)(x) << CCMD_CIRG_SHIFT)
#define   CCMD_CAIG_MASK	0x3
#define   CCMD_CAIG_SHIFT	59
#define   CCMD_FM_MASK		0x3
#define   CCMD_FM_SHIFT		32
#define   CCMD_FM(x)		(((uint64_t)(x) << CCMD_FM_SHIFT))
#define   CCMD_SID_MASK		0xFFFF
#define   CCMD_SID_SHIFT	8
#define   CCMD_SID(x)		(((x) << CCMD_SID_SHIFT))
#define   CCMD_DID_MASK		0xFFFF
#define   CCMD_DID_SHIFT	0
#define   CCMD_DID(x)		(((x) << CCMD_DID_SHIFT))

#define CIG_GLOBAL		CCMD_CIRG(CTX_GLOBAL)
#define CIG_DOMAIN		CCMD_CIRG(CTX_DOMAIN)
#define CIG_DEVICE		CCMD_CIRG(CTX_DEVICE)


#define DMAR_FSTS_REG		0x34	/* 32:Fault Status register */
#define   FSTS_FRI_MASK		0xFF
#define   FSTS_FRI_SHIFT	8
#define   FSTS_PRO		(1LL << 7)
#define   FSTS_ITE		(1LL << 6)
#define   FSTS_ICE		(1LL << 5)
#define   FSTS_IQE		(1LL << 4)
#define   FSTS_APF		(1LL << 3)
#define   FSTS_APO		(1LL << 2)
#define   FSTS_PPF		(1LL << 1)
#define   FSTS_PFO		(1LL << 0)

#define DMAR_FECTL_REG		0x38	/* 32:Fault control register */
#define   FECTL_IM		(1LL << 31)
#define   FECTL_IP		(1LL << 30)

#define FRCD_HI_F		(1LL << (127-64))
#define FRCD_HI_T		(1LL << (126-64))
#define FRCD_HI_AT_MASK		0x3
#define FRCD_HI_AT_SHIFT	(124-64)
#define FRCD_HI_PV_MASK		0xFFFFF
#define FRCD_HI_PV_SHIFT	(104-64)
#define FRCD_HI_FR_MASK		0xFF
#define FRCD_HI_FR_SHIFT	(96-64)
#define FRCD_HI_PP		(1LL << (95-64))

#define FRCD_HI_SID_MASK	0xFF
#define FRCD_HI_SID_SHIFT	0
#define FRCD_HI_BUS_SHIFT	8
#define FRCD_HI_BUS_MASK	0xFF
#define FRCD_HI_DEV_SHIFT	3
#define FRCD_HI_DEV_MASK	0x1F
#define FRCD_HI_FUN_SHIFT	0
#define FRCD_HI_FUN_MASK	0x7

#define DMAR_IOTLB_REG(x)	(ecap_iro((x)->ecap) + 8)
#define DMAR_IVA_REG(x)		(ecap_iro((x)->ecap) + 0)

#define DMAR_FRIH_REG(x,i)	(cap_fro((x)->cap) + 16*(i) + 8)
#define DMAR_FRIL_REG(x,i)	(cap_fro((x)->cap) + 16*(i) + 0)

#define IOTLB_IVT		(1LL << 63)
#define IOTLB_IIRG_MASK		0x3
#define IOTLB_IIRG_SHIFT	60
#define IOTLB_IIRG(x)		((uint64_t)(x) << IOTLB_IIRG_SHIFT)
#define IOTLB_IAIG_MASK		0x3
#define IOTLB_IAIG_SHIFT	57
#define IOTLB_DR		(1LL << 49)
#define IOTLB_DW		(1LL << 48)
#define IOTLB_DID_MASK		0xFFFF
#define IOTLB_DID_SHIFT		32
#define IOTLB_DID(x)		((uint64_t)(x) << IOTLB_DID_SHIFT)

#define IIG_GLOBAL	IOTLB_IIRG(IOTLB_GLOBAL)
#define IIG_DOMAIN	IOTLB_IIRG(IOTLB_DOMAIN)
#define IIG_PAGE	IOTLB_IIRG(IOTLB_PAGE)

#define DMAR_IQH_REG	0x80	/* 64:Invalidation queue head register */
#define DMAR_IQT_REG	0x88	/* 64:Invalidation queue tail register */
#define DMAR_IQA_REG	0x90	/* 64:Invalidation queue addr register */
#define IQA_QS_256	0	/* 256 entries */
#define IQA_QS_512	1	/* 512 */
#define IQA_QS_1K	2	/* 1024 */
#define IQA_QS_2K	3	/* 2048 */
#define IQA_QS_4K	4	/* 4096 */
#define IQA_QS_8K	5	/* 8192 */
#define IQA_QS_16K	6	/* 16384 */
#define IQA_QS_32K	7	/* 32768 */

/* Read-Modify-Write helpers */
static inline void
iommu_rmw32(void *ov, uint32_t mask, uint32_t shift, uint32_t nv)
{
	*(uint32_t *)ov &= ~(mask << shift);
	*(uint32_t *)ov |= (nv & mask) << shift;
}

static inline void
iommu_rmw64(void *ov, uint32_t mask, uint32_t shift, uint64_t nv)
{
	*(uint64_t *)ov &= ~(mask << shift);
	*(uint64_t *)ov |= (nv & mask) << shift;
}

/*
 * Root Entry: one per bus (256 x 128 bit = 4k)
 *   0        = Present
 *   1:11     = Reserved
 *   12:HAW-1 = Context Table Pointer
 *   HAW:63   = Reserved
 *   64:127   = Reserved
 */
#define ROOT_P	(1L << 0)
struct root_entry {
	uint64_t		lo;
	uint64_t		hi;
};

/* Check if root entry is valid */
static inline bool
root_entry_is_valid(struct root_entry *re)
{
	return (re->lo & ROOT_P);
}

/*
 * Context Entry: one per devfn (256 x 128 bit = 4k)
 *   0      = Present
 *   1      = Fault Processing Disable
 *   2:3    = Translation Type
 *   4:11   = Reserved
 *   12:63  = Second Level Page Translation
 *   64:66  = Address Width (# PTE levels)
 *   67:70  = Ignore
 *   71     = Reserved
 *   72:87  = Domain ID
 *   88:127 = Reserved
 */
#define CTX_P		(1L << 0)
#define CTX_FPD		(1L << 1)
#define CTX_T_MASK	0x3
#define CTX_T_SHIFT	2
enum {
	CTX_T_MULTI,
	CTX_T_IOTLB,
	CTX_T_PASSTHRU
};

#define CTX_H_AW_MASK	0x7
#define CTX_H_AW_SHIFT	0
#define CTX_H_USER_MASK 0xF
#define CTX_H_USER_SHIFT 3
#define CTX_H_DID_MASK	0xFFFF
#define CTX_H_DID_SHIFT	8

struct context_entry {
	uint64_t		lo;
	uint64_t		hi;
};

/* Set fault processing enable/disable */
static inline void
context_set_fpd(struct context_entry *ce, int enable)
{
	ce->lo &= ~CTX_FPD;
	if (enable)
		ce->lo |= CTX_FPD;
}

/* Set context entry present */
static inline void
context_set_present(struct context_entry *ce)
{
	ce->lo |= CTX_P;
}

/* Set Second Level Page Table Entry PA */
static inline void
context_set_slpte(struct context_entry *ce, paddr_t slpte)
{
	ce->lo &= VTD_PAGE_MASK;
	ce->lo |= (slpte & ~VTD_PAGE_MASK);
}

/* Set translation type */
static inline void
context_set_translation_type(struct context_entry *ce, int tt)
{
	ce->lo &= ~(CTX_T_MASK << CTX_T_SHIFT);
	ce->lo |= ((tt & CTX_T_MASK) << CTX_T_SHIFT);
}

/* Set Address Width (# of Page Table levels) */
static inline void
context_set_address_width(struct context_entry *ce, int lvl)
{
	ce->hi &= ~(CTX_H_AW_MASK << CTX_H_AW_SHIFT);
	ce->hi |= ((lvl & CTX_H_AW_MASK) << CTX_H_AW_SHIFT);
}

/* Set domain ID */
static inline void
context_set_domain_id(struct context_entry *ce, int did)
{
	ce->hi &= ~(CTX_H_DID_MASK << CTX_H_DID_SHIFT);
	ce->hi |= ((did & CTX_H_DID_MASK) << CTX_H_DID_SHIFT);
}

/* Get Second Level Page Table PA */
static inline uint64_t
context_pte(struct context_entry *ce)
{
	return (ce->lo & ~VTD_PAGE_MASK);
}

/* Get translation type */
static inline int
context_translation_type(struct context_entry *ce)
{
	return (ce->lo >> CTX_T_SHIFT) & CTX_T_MASK;
}

/* Get domain ID */
static inline int
context_domain_id(struct context_entry *ce)
{
	return (ce->hi >> CTX_H_DID_SHIFT) & CTX_H_DID_MASK;
}

/* Get Address Width */
static inline int
context_address_width(struct context_entry *ce)
{
	return VTD_LEVELTOAW((ce->hi >> CTX_H_AW_SHIFT) & CTX_H_AW_MASK);
}

/* Check if context entry is valid */
static inline bool
context_entry_is_valid(struct context_entry *ce)
{
	return (ce->lo & CTX_P);
}

/* User-available bits in context entry */
static inline int
context_user(struct context_entry *ce)
{
	return (ce->hi >> CTX_H_USER_SHIFT) & CTX_H_USER_MASK;
}

static inline void
context_set_user(struct context_entry *ce, int v)
{
	ce->hi &= ~(CTX_H_USER_MASK << CTX_H_USER_SHIFT);
	ce->hi |=  ((v & CTX_H_USER_MASK) << CTX_H_USER_SHIFT);
}

/*
 * Fault entry
 *   0..HAW-1 = Fault address
 *   HAW:63   = Reserved
 *   64:71    = Source ID
 *   96:103   = Fault Reason
 *   104:123  = PV
 *   124:125  = Address Translation type
 *   126      = Type (0 = Read, 1 = Write)
 *   127      = Fault bit
 */
struct fault_entry {
	uint64_t	lo;
	uint64_t	hi;
};

/* PTE Entry: 512 x 64-bit = 4k */
#define PTE_P	(1L << 0)
#define PTE_R	0x00
#define PTE_W	(1L << 1)
#define PTE_US  (1L << 2)
#define PTE_PWT (1L << 3)
#define PTE_PCD (1L << 4)
#define PTE_A   (1L << 5)
#define PTE_D   (1L << 6)
#define PTE_PAT (1L << 7)
#define PTE_G   (1L << 8)
#define PTE_EA  (1L << 10)
#define PTE_XD  (1LL << 63)

/* PDE Level entry */
#define PTE_PS  (1L << 7)

/* PDPE Level entry */

/* ----------------------------------------------------------------
 * 5555555444444444333333333222222222111111111000000000------------
 * [PML4 ->] PDPE.1GB
 * [PML4 ->] PDPE.PDE -> PDE.2MB
 * [PML4 ->] PDPE.PDE -> PDE -> PTE
 * GAW0 = (12.20) (PTE)
 * GAW1 = (21.29) (PDE)
 * GAW2 = (30.38) (PDPE)
 * GAW3 = (39.47) (PML4)
 * GAW4 = (48.57) (n/a)
 * GAW5 = (58.63) (n/a)
 */
struct pte_entry {
	uint64_t	val;
};

/*
 * Queued Invalidation entry
 *  0:3   = 01h
 *  4:5   = Granularity
 *  6:15  = Reserved
 *  16:31 = Domain ID
 *  32:47 = Source ID
 *  48:49 = FM
 */

/* Invalidate Context Entry */
#define QI_CTX_DID_MASK		0xFFFF
#define QI_CTX_DID_SHIFT	16
#define QI_CTX_SID_MASK		0xFFFF
#define QI_CTX_SID_SHIFT	32
#define QI_CTX_FM_MASK		0x3
#define QI_CTX_FM_SHIFT		48
#define QI_CTX_IG_MASK		0x3
#define QI_CTX_IG_SHIFT		4
#define QI_CTX_DID(x)		(((uint64_t)(x) << QI_CTX_DID_SHIFT))
#define QI_CTX_SID(x)		(((uint64_t)(x) << QI_CTX_SID_SHIFT))
#define QI_CTX_FM(x)		(((uint64_t)(x) << QI_CTX_FM_SHIFT))

#define QI_CTX_IG_GLOBAL	(CTX_GLOBAL << QI_CTX_IG_SHIFT)
#define QI_CTX_IG_DOMAIN	(CTX_DOMAIN << QI_CTX_IG_SHIFT)
#define QI_CTX_IG_DEVICE	(CTX_DEVICE << QI_CTX_IG_SHIFT)

/* Invalidate IOTLB Entry */
#define QI_IOTLB_DID_MASK	0xFFFF
#define QI_IOTLB_DID_SHIFT	16
#define QI_IOTLB_IG_MASK	0x3
#define QI_IOTLB_IG_SHIFT	4
#define QI_IOTLB_DR		(1LL << 6)
#define QI_IOTLB_DW		(1LL << 5)
#define QI_IOTLB_DID(x)		(((uint64_t)(x) << QI_IOTLB_DID_SHIFT))

#define QI_IOTLB_IG_GLOBAL	(1 << QI_IOTLB_IG_SHIFT)
#define QI_IOTLB_IG_DOMAIN	(2 << QI_IOTLB_IG_SHIFT)
#define QI_IOTLB_IG_PAGE	(3 << QI_IOTLB_IG_SHIFT)

/* QI Commands */
#define QI_CTX		0x1
#define QI_IOTLB	0x2
#define QI_DEVTLB	0x3
#define QI_INTR		0x4
#define QI_WAIT		0x5
#define QI_EXTTLB	0x6
#define QI_PAS		0x7
#define QI_EXTDEV	0x8

struct qi_entry {
	uint64_t	lo;
	uint64_t	hi;
};

enum {
	CTX_GLOBAL = 1,
	CTX_DOMAIN,
	CTX_DEVICE,

	IOTLB_GLOBAL = 1,
	IOTLB_DOMAIN,
	IOTLB_PAGE,
};

enum {
	VTD_FAULT_ROOT_P = 0x1,         /* P field in root entry is 0 */
	VTD_FAULT_CTX_P = 0x2,          /* P field in context entry is 0 */
	VTD_FAULT_CTX_INVAL = 0x3,      /* context AW/TT/SLPPTR invalid */
	VTD_FAULT_LIMIT = 0x4,          /* Address is outside of MGAW */
	VTD_FAULT_WRITE = 0x5,          /* Address-translation fault, non-writable */
	VTD_FAULT_READ = 0x6,           /* Address-translation fault, non-readable */
	VTD_FAULT_PTE_INVAL = 0x7,      /* page table hw access error */
	VTD_FAULT_ROOT_INVAL = 0x8,     /* root table hw access error */
	VTD_FAULT_CTX_TBL_INVAL = 0x9,  /* context entry hw access error */
	VTD_FAULT_ROOT_RESERVED = 0xa,  /* non-zero reserved field in root entry */
	VTD_FAULT_CTX_RESERVED = 0xb,   /* non-zero reserved field in context entry */
	VTD_FAULT_PTE_RESERVED = 0xc,   /* non-zero reserved field in paging entry */
	VTD_FAULT_CTX_TT = 0xd,         /* invalid translation type */
};

#endif

void	acpidmar_pci_hook(pci_chipset_tag_t, struct pci_attach_args *);
void	dmar_ptmap(bus_dma_tag_t, bus_addr_t);

#define __EXTRACT(v,m) (((v) >> m##_SHIFT) & m##_MASK)