Support 2TB phys mem

This change expands the direct map to 4 slots (512GB each), to support
machines with up to 2TB physical memory. Should further expansion be
required, this change provides the means to do that with a single #define
change.

with help from and ok guenther

3 files changed, 80 insertions, 16 deletions

diff --git a/sys/arch/amd64/amd64/machdep.c b/sys/arch/amd64/amd64/machdep.c
index 0990d984320..3e10b2e377a 100644
--- a/sys/arch/amd64/amd64/machdep.c
+++ b/sys/arch/amd64/amd64/machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: machdep.c,v 1.253 2019/01/19 20:45:06 tedu Exp $	*/
+/*	$OpenBSD: machdep.c,v 1.254 2019/01/21 06:18:37 mlarkin Exp $	*/
 /*	$NetBSD: machdep.c,v 1.3 2003/05/07 22:58:18 fvdl Exp $	*/
 
 /*-
@@ -1366,6 +1366,7 @@ init_x86_64(paddr_t first_avail)
 	struct region_descriptor region;
 	bios_memmap_t *bmp;
 	int x, ist;
+	uint64_t max_dm_size = ((uint64_t)512 * NUM_L4_SLOT_DIRECT) << 30;
 
 	cpu_init_msrs(&cpu_info_primary);
 
@@ -1496,11 +1497,11 @@ init_x86_64(paddr_t first_avail)
 		}
 
 		/*
-		 * The direct map is limited to 512GB of memory, so
-		 * discard anything above that.
+		 * The direct map is limited to 512GB * NUM_L4_SLOT_DIRECT of
+		 * memory, so discard anything above that.
 		 */
-		if (e1 >= (uint64_t)512*1024*1024*1024) {
-			e1 = (uint64_t)512*1024*1024*1024;
+		if (e1 >= max_dm_size) {
+			e1 = max_dm_size;
 			if (s1 > e1)
 				continue;
 		}
diff --git a/sys/arch/amd64/amd64/pmap.c b/sys/arch/amd64/amd64/pmap.c
index 2cb9646ccca..ed9388d721e 100644
--- a/sys/arch/amd64/amd64/pmap.c
+++ b/sys/arch/amd64/amd64/pmap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pmap.c,v 1.126 2019/01/21 05:40:11 mlarkin Exp $	*/
+/*	$OpenBSD: pmap.c,v 1.127 2019/01/21 06:18:37 mlarkin Exp $	*/
 /*	$NetBSD: pmap.c,v 1.3 2003/05/08 18:13:13 thorpej Exp $	*/
 
 /*
@@ -560,10 +560,11 @@ pmap_bootstrap(paddr_t first_avail, paddr_t max_pa)
 {
 	vaddr_t kva_start = VM_MIN_KERNEL_ADDRESS;
 	struct pmap *kpm;
-	int i;
+	int curslot, i, j, k, p;
 	long ndmpdp;
-	paddr_t dmpd, dmpdp;
+	paddr_t dmpd, dmpdp, start_cur;
 	vaddr_t kva, kva_end;
+	pt_entry_t *pt, *pml2;
 
 	/*
 	 * define the boundaries of the managed kernel virtual address
@@ -657,10 +658,16 @@ pmap_bootstrap(paddr_t first_avail, paddr_t max_pa)
 	 * we map the rest if it exists. We actually use the direct map
 	 * here to set up the page tables, we're assuming that we're still
 	 * operating in the lower 4GB of memory.
+	 *
+	 * Map (up to) the first 512GB of physical memory first. This part
+	 * is handled differently than physical memory > 512GB since we have
+	 * already mapped part of this range in locore0.
 	 */
 	ndmpdp = (max_pa + NBPD_L3 - 1) >> L3_SHIFT;
 	if (ndmpdp < NDML2_ENTRIES)
 		ndmpdp = NDML2_ENTRIES;		/* At least 4GB */
+	if (ndmpdp > 512)
+		ndmpdp = 512;			/* At most 512GB */
 
 	dmpdp = kpm->pm_pdir[PDIR_SLOT_DIRECT] & PG_FRAME;
 
@@ -692,6 +699,57 @@ pmap_bootstrap(paddr_t first_avail, paddr_t max_pa)
 	kpm->pm_pdir[PDIR_SLOT_DIRECT] = dmpdp | PG_V | PG_KW | PG_U |
 	    PG_M | pg_nx;
 
+	/* Map any remaining physical memory > 512GB */	
+	for (curslot = 1 ; curslot < NUM_L4_SLOT_DIRECT ; curslot++) {
+		/*
+		 * Start of current range starts at PA (curslot) * 512GB
+		 */
+		start_cur = (paddr_t)(curslot * NBPD_L4);
+		if (max_pa > start_cur) {
+			/* Next 512GB, new PML4e and PML3 page */
+			dmpd = first_avail; first_avail += PAGE_SIZE;
+			pt = (pt_entry_t *)PMAP_DIRECT_MAP(dmpd);
+			kpm->pm_pdir[PDIR_SLOT_DIRECT + curslot] = dmpd |
+			    PG_KW | PG_V | PG_U | PG_M | pg_nx;
+
+			/* How many 1GB entries remain? */
+			p = ((max_pa - start_cur) >> L3_SHIFT);
+			if (max_pa & L2_MASK)
+				p++;
+
+			if (p > NPDPG)
+				p = NPDPG;
+
+			/* Allocate 'p' PML2 pages and populate */
+			for (i = 0; i < p; i++) {
+				dmpd = first_avail; first_avail += PAGE_SIZE;
+				pt[i] = dmpd |
+				    PG_RW | PG_V | PG_U | PG_M | pg_nx;
+
+				pml2 = (pt_entry_t *)PMAP_DIRECT_MAP(dmpd);
+			
+				/*
+				 * All PML2 pages except the last one will be
+				 * filled
+				 */	
+				if (i == p - 1) {
+					k = ((max_pa & L2_MASK) >> L2_SHIFT);
+					if (max_pa & L1_MASK)
+						k++;
+				} else
+					k = NPDPG;
+
+				for (j = 0; j < k; j++) {
+					pml2[j] = curslot * NBPD_L4 +
+					    (uint64_t)i * NBPD_L3 +
+					    (uint64_t)j * NBPD_L2;
+					pml2[j] |= PG_RW | PG_V | pg_g_kern |
+					    PG_U | PG_M | pg_nx | PG_PS;
+				}
+			}
+		}
+	}
+
 	tlbflush();
 
 	msgbuf_vaddr = virtual_avail;
@@ -1108,10 +1166,11 @@ void
 pmap_pdp_ctor(pd_entry_t *pdir)
 {
 	paddr_t pdirpa;
-	int npde;
+	int npde, i;
+	struct pmap *kpm = pmap_kernel();
 
 	/* fetch the physical address of the page directory. */
-	(void) pmap_extract(pmap_kernel(), (vaddr_t) pdir, &pdirpa);
+	(void) pmap_extract(kpm, (vaddr_t) pdir, &pdirpa);
 
 	/* zero init area */
 	memset(pdir, 0, PDIR_SLOT_PTE * sizeof(pd_entry_t));
@@ -1129,7 +1188,8 @@ pmap_pdp_ctor(pd_entry_t *pdir)
 	memset(&pdir[PDIR_SLOT_KERN + npde], 0,
 	    (NTOPLEVEL_PDES - (PDIR_SLOT_KERN + npde)) * sizeof(pd_entry_t));
 
-	pdir[PDIR_SLOT_DIRECT] = pmap_kernel()->pm_pdir[PDIR_SLOT_DIRECT];
+	for (i = 0; i < NUM_L4_SLOT_DIRECT; i++)
+		pdir[PDIR_SLOT_DIRECT + i] = kpm->pm_pdir[PDIR_SLOT_DIRECT + i];
 
 #if VM_MIN_KERNEL_ADDRESS != KERNBASE
 	pdir[pl4_pi(KERNBASE)] = PDP_BASE[pl4_pi(KERNBASE)];
diff --git a/sys/arch/amd64/include/pmap.h b/sys/arch/amd64/include/pmap.h
index e1d29f1e8b8..e872880fe3c 100644
--- a/sys/arch/amd64/include/pmap.h
+++ b/sys/arch/amd64/include/pmap.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pmap.h,v 1.72 2019/01/19 04:07:12 mlarkin Exp $	*/
+/*	$OpenBSD: pmap.h,v 1.73 2019/01/21 06:18:37 mlarkin Exp $	*/
 /*	$NetBSD: pmap.h,v 1.1 2003/04/26 18:39:46 fvdl Exp $	*/
 
 /*
@@ -94,7 +94,7 @@
  * The other obvious difference from i386 is that it has a direct map of all
  * physical memory in the VA range:
  *
- *     0xffffff0000000000 - 0xffffff7fffffffff
+ *     0xfffffd8000000000 - 0xffffff7fffffffff
  *
  * The direct map is used in some cases to access PTEs of non-current pmaps.
  *
@@ -104,7 +104,7 @@
  *  |         Kernel Image            |
  *  +---------------------------------+ 0xffffff8000000000
  *  |         Direct Map              |
- *  +---------------------------------+ 0xffffff0000000000
+ *  +---------------------------------+ 0xfffffd8000000000
  *  ~                                 ~
  *  |                                 |
  *  |         Kernel Space            |
@@ -141,7 +141,8 @@
 #define L4_SLOT_PTE		255
 #define L4_SLOT_KERN		256
 #define L4_SLOT_KERNBASE	511
-#define L4_SLOT_DIRECT		510
+#define NUM_L4_SLOT_DIRECT	4
+#define L4_SLOT_DIRECT		(L4_SLOT_KERNBASE - NUM_L4_SLOT_DIRECT)
 
 #define PDIR_SLOT_KERN		L4_SLOT_KERN
 #define PDIR_SLOT_PTE		L4_SLOT_PTE
@@ -157,7 +158,8 @@
 
 #define PTE_BASE  ((pt_entry_t *) (L4_SLOT_PTE * NBPD_L4))
 #define PMAP_DIRECT_BASE	(VA_SIGN_NEG((L4_SLOT_DIRECT * NBPD_L4)))
-#define PMAP_DIRECT_END		(VA_SIGN_NEG(((L4_SLOT_DIRECT + 1) * NBPD_L4)))
+#define PMAP_DIRECT_END		(VA_SIGN_NEG(((L4_SLOT_DIRECT + \
+    NUM_L4_SLOT_DIRECT) * NBPD_L4)))
 
 #define L1_BASE		PTE_BASE
 
@@ -178,6 +180,7 @@
 #define NKL3_KIMG_ENTRIES	1
 #define NKL2_KIMG_ENTRIES	64
 
+/* number of pages of direct map entries set up by locore0.S */
 #define NDML4_ENTRIES		1
 #define NDML3_ENTRIES		1
 #define NDML2_ENTRIES		4	/* 4GB */