as seen in freebsd: asm pagezero implementations, but use a fn pointer.

the sse2 version cuts ZOD fault time in half.
suggestions mickey deraadt.  many many testers. ok deraadt.

3 files changed, 113 insertions, 8 deletions

diff --git a/sys/arch/i386/i386/locore.s b/sys/arch/i386/i386/locore.s
index 3951db7a75f..37ed4107fde 100644
--- a/sys/arch/i386/i386/locore.s
+++ b/sys/arch/i386/i386/locore.s
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.s,v 1.75 2003/11/17 14:55:58 miod Exp $	*/
+/*	$OpenBSD: locore.s,v 1.76 2004/01/29 19:01:53 tedu Exp $	*/
 /*	$NetBSD: locore.s,v 1.145 1996/05/03 19:41:19 christos Exp $	*/
 
 /*-
@@ -1972,3 +1972,70 @@ ENTRY(bzero)
 
 	popl	%edi
 	ret
+
+#if defined(I686_CPU) && !defined(SMALL_KERNEL)
+ENTRY(sse2_pagezero)
+	pushl	%ebx
+	movl	8(%esp),%ecx
+	movl	%ecx,%eax
+	addl	$4096,%eax
+	xor	%ebx,%ebx
+1:
+	movnti	%ebx,(%ecx)
+	addl	$4,%ecx
+	cmpl	%ecx,%eax
+	jne	1b
+	sfence
+	popl	%ebx
+	ret
+
+ENTRY(i686_pagezero)
+	pushl	%edi
+	pushl	%ebx
+
+	movl	12(%esp), %edi
+	movl	$1024, %ecx
+	cld
+
+	ALIGN_TEXT
+1:
+	xorl	%eax, %eax
+	repe
+	scasl
+	jnz	2f
+
+	popl	%ebx
+	popl	%edi
+	ret
+
+	ALIGN_TEXT
+
+2:
+	incl	%ecx
+	subl	$4, %edi
+
+	movl	%ecx, %edx
+	cmpl	$16, %ecx
+
+	jge	3f
+
+	movl	%edi, %ebx
+	andl	$0x3f, %ebx
+	shrl	%ebx
+	shrl	%ebx
+	movl	$16, %ecx
+	subl	%ebx, %ecx
+
+3:
+	subl	%ecx, %edx
+	rep
+	stosl
+
+	movl	%edx, %ecx
+	testl	%edx, %edx
+	jnz	1b
+
+	popl	%ebx
+	popl	%edi
+	ret
+#endif
diff --git a/sys/arch/i386/i386/machdep.c b/sys/arch/i386/i386/machdep.c
index e7ae0b12894..e1de76e4b89 100644
--- a/sys/arch/i386/i386/machdep.c
+++ b/sys/arch/i386/i386/machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: machdep.c,v 1.264 2004/01/29 02:14:52 tom Exp $	*/
+/*	$OpenBSD: machdep.c,v 1.265 2004/01/29 19:01:54 tedu Exp $	*/
 /*	$NetBSD: machdep.c,v 1.214 1996/11/10 03:16:17 thorpej Exp $	*/
 
 /*-
@@ -314,6 +314,7 @@ int allowaperture = 0;
 
 void	winchip_cpu_setup(const char *, int, int);
 void	amd_family5_setup(const char *, int, int);
+void	amd_family6_setup(const char *, int, int);
 void	cyrix3_cpu_setup(const char *, int, int);
 void	cyrix6x86_cpu_setup(const char *, int, int);
 void	natsem6x86_cpu_setup(const char *, int, int);
@@ -805,7 +806,7 @@ const struct cpu_cpuid_nameclass i386_cpuid_cpus[] = {
 				0, 0, 0, 0, 0,
 				"K7"		/* Default */
 			},
-			NULL
+			amd_family6_setup
 		},
 		/* Family 7 */
 		{
@@ -848,7 +849,7 @@ const struct cpu_cpuid_nameclass i386_cpuid_cpus[] = {
 				0, 0, 0, 0, 0, 0, 0, 0,
 				"AMD64"			/* DEFAULT */
 			},
-			NULL
+			amd_family6_setup
 		} }
 	},
 	{
@@ -1493,6 +1494,13 @@ cyrix3_cpu_setup(cpu_device, model, step)
 {
 #if defined(I686_CPU)
 	unsigned int val;
+#if !defined(SMALL_KERNEL)
+	extern void (*pagezero)(void *, size_t);
+	extern void i686_pagezero(void *, size_t);
+
+	pagezero = i686_pagezero;
+#endif
+
 
 	switch (model) {
 	case 6: /* C3 Samuel 1 */
@@ -1666,6 +1674,23 @@ amd_family5_setup(cpu_device, model, step)
 }
 
 void
+amd_family6_setup(cpu_device, model, step)
+	const char *cpu_device;
+	int model, step;
+{
+#if !defined(SMALL_KERNEL) && defined (I686_CPU)
+	extern void (*pagezero)(void *, size_t);
+	extern void sse2_pagezero(void *, size_t);
+	extern void i686_pagezero(void *, size_t);
+
+	if (cpu_feature & CPUID_SIMD2)
+		pagezero = sse2_pagezero;
+	else
+		pagezero = i686_pagezero;
+#endif
+}
+
+void
 intel686_cpu_setup(cpu_device, model, step)
 	const char *cpu_device;
 	int model, step;
@@ -1707,6 +1732,18 @@ intel686_cpu_setup(cpu_device, model, step)
 	} else if ((cpu_feature & (CPUID_ACPI | CPUID_TM)) ==
 	    (CPUID_ACPI | CPUID_TM))
 		p4tcc_init(model, step);
+
+	{
+	extern void (*pagezero)(void *, size_t);
+	extern void sse2_pagezero(void *, size_t);
+	extern void i686_pagezero(void *, size_t);
+
+	if (cpu_feature & CPUID_SIMD2)
+		pagezero = sse2_pagezero;
+	else
+		pagezero = i686_pagezero;
+	pagezero = bzero;
+	}
 #endif
 }
 
diff --git a/sys/arch/i386/i386/pmap.c b/sys/arch/i386/i386/pmap.c
index 56301ed68c8..622827a5879 100644
--- a/sys/arch/i386/i386/pmap.c
+++ b/sys/arch/i386/i386/pmap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pmap.c,v 1.73 2003/05/23 16:33:35 mpech Exp $	*/
+/*	$OpenBSD: pmap.c,v 1.74 2004/01/29 19:01:54 tedu Exp $	*/
 /*	$NetBSD: pmap.c,v 1.91 2000/06/02 17:46:37 thorpej Exp $	*/
 
 /*
@@ -2057,6 +2057,7 @@ pmap_virtual_space(startp, endp)
 /*
  * pmap_zero_page: zero a page
  */
+void (*pagezero)(void *, size_t) = bzero;
 
 void
 pmap_zero_page(struct vm_page *pg)
@@ -2070,7 +2071,7 @@ pmap_zero_page(struct vm_page *pg)
 #endif
 
 	*zero_pte = (pa & PG_FRAME) | PG_V | PG_RW;	/* map in */
-	bzero(zerop, NBPG);				/* zero */
+	pagezero(zerop, PAGE_SIZE);				/* zero */
 	*zero_pte = 0;				/* zap! */
 	pmap_update_pg((vaddr_t)zerop);		/* flush TLB */
 	simple_unlock(&pmap_zero_page_lock);
@@ -2090,7 +2091,7 @@ pmap_zero_phys(paddr_t pa)
 #endif
 
 	*zero_pte = (pa & PG_FRAME) | PG_V | PG_RW;	/* map in */
-	bzero(zerop, NBPG);				/* zero */
+	pagezero(zerop, PAGE_SIZE);				/* zero */
 	*zero_pte = 0;				/* zap! */
 	pmap_update_pg((vaddr_t)zerop);		/* flush TLB */
 	simple_unlock(&pmap_zero_page_lock);
@@ -2112,7 +2113,7 @@ pmap_zero_page_uncached(pa)
 
 	*zero_pte = (pa & PG_FRAME) | PG_V | PG_RW |	/* map in */
 	    ((cpu_class != CPUCLASS_386) ? PG_N : 0);
-	memset(zerop, 0, NBPG);				/* zero */
+	pagezero(zerop, PAGE_SIZE);				/* zero */
 	*zero_pte = 0;					/* zap! */
 	pmap_update_pg((vaddr_t)zerop);			/* flush TLB */
 	simple_unlock(&pmap_zero_page_lock);