Fix a number of issues with vesabios discovered by miod@

1. Add bounds checking to vga_pci_mmap, fixing a potential security issue.
Limit the amount of memory to what the vbe info structure says the card
card has. This should be nicely refined to limit to the amount of memory
needed by the active resolution and depth but this will do for the time
being.

2. Fix several places in vesafb.c where the return of kvm86_bios_addpage()
isnt being checked. Also ensure that if vesafb_get_mode_info() fails that it
cleans up after itself by releasing the page it added.

3. Correct the range checks in vesafb_putcmap and vesafb_getcmap, harmonize
code with similar code found else where.

ok miod@

1 files changed, 16 insertions, 11 deletions

diff --git a/sys/dev/vesa/vesafb.c b/sys/dev/vesa/vesafb.c
index 701a1157fb1..9d0598572ce 100644
--- a/sys/dev/vesa/vesafb.c
+++ b/sys/dev/vesa/vesafb.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: vesafb.c,v 1.2 2006/12/02 20:20:55 matthieu Exp $ */
+/* $OpenBSD: vesafb.c,v 1.3 2007/01/28 20:28:50 gwk Exp $ */
 
 /*-
  * Copyright (c) 2006 Jared D. McNeill <jmcneill@invisible.ca>
@@ -112,7 +112,10 @@ vesafb_get_mode_info(struct vga_pci_softc *sc, int mode,
 	unsigned char *buf;
 	int res;
 
-	buf = kvm86_bios_addpage(0x2000);
+	if ((buf = kvm86_bios_addpage(0x2000)) == NULL) {
+		printf("%s: kvm86_bios_addpage(0x2000) failed.\n");
+		return;
+	}
 	memset(&tf, 0, sizeof(struct trapframe));
 	tf.tf_eax = 0x4f01; /* function code */
 	tf.tf_ecx = mode;
@@ -123,8 +126,9 @@ vesafb_get_mode_info(struct vga_pci_softc *sc, int mode,
 	if (res || (tf.tf_eax & 0xff) != 0x4f) {
 		printf("%s: vbecall: res=%d, ax=%x\n",
 		    sc->sc_dev.dv_xname, res, tf.tf_eax);
-		printf("%s: error getting info for mode %04x\n",
+		printf("%s: error getting info for mode %05x\n",
 		    sc->sc_dev.dv_xname, mode);
+		kvm86_bios_delpage(0x2000, buf);
 		return;
 	}
 	memcpy(mi, buf, sizeof(struct modeinfoblock));
@@ -138,8 +142,10 @@ vesafb_set_palette(struct vga_pci_softc *sc, int reg, struct paletteentry pe)
 	int res;
 	char *buf;
 
-	buf = kvm86_bios_addpage(0x2000);
-
+	if ((buf = kvm86_bios_addpage(0x2000)) == NULL) {
+		printf("%s: kvm86_bios_addpage(0x2000) failed.\n");
+		return;
+	}
 	/*
 	 * this function takes 8 bit per palette as input, but we're
 	 * working in 6 bit mode here
@@ -229,7 +235,7 @@ vesafb_putcmap(struct vga_pci_softc *sc, struct wsdisplay_cmap *cm)
 	idx = cm->index;
 	cnt = cm->count;
 
-	if (idx >= 255 || cnt > 256 || idx + cnt > 256)
+	if (idx >= 256 || cnt > 256 - idx)
 		return EINVAL;
 
 	rv = copyin(cm->red, &r[idx], cnt);
@@ -272,7 +278,7 @@ vesafb_getcmap(struct vga_pci_softc *sc, struct wsdisplay_cmap *cm)
 	idx = cm->index;
 	cnt = cm->count;
 
-	if (idx >= 255 || cnt > 256 || idx + cnt > 256)
+	if (idx >= 256 || cnt > 256 - idx)
 		return EINVAL;
 
 	rv = copyout(&sc->sc_cmap_red[idx], cm->red, cnt);
@@ -292,8 +298,7 @@ static int
 vesafb_getdepthflag(struct modeinfoblock *mi)
 {
 	int bpp, depth;
-	
-	
+
 	depth = mi->RedMaskSize + mi->GreenMaskSize + mi->BlueMaskSize;
 	bpp = mi->BitsPerPixel;
 	switch (depth) {
@@ -326,10 +331,10 @@ vesafb_get_supported_depth(struct vga_pci_softc *sc)
 	struct modeinfoblock mi;
 
 	depths = 0;
-	
+
 	if (vesabios_softc == NULL || vesabios_softc->sc_nmodes == 0)
 		return 0;
-	
+
 	for (i = 0; i < vesabios_softc->sc_nmodes; i++) {
 		vesafb_get_mode_info(sc, vesabios_softc->sc_modes[i], &mi);
 		depths |= vesafb_getdepthflag(&mi);