path: root/sys/arch/loongson/dev/bonito.c

/*	$OpenBSD: bonito.c,v 1.34 2018/02/24 11:42:30 visa Exp $	*/
/*	$NetBSD: bonito_mainbus.c,v 1.11 2008/04/28 20:23:10 martin Exp $	*/
/*	$NetBSD: bonito_pci.c,v 1.5 2008/04/28 20:23:28 martin Exp $	*/

/*
 * Copyright (c) 2009, 2010 Miodrag Vallat.
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */
/*-
 * Copyright (c) 2001 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Jason R. Thorpe.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * PCI configuration space support for the Loongson PCI and memory controller
 * chip, which is derived from the Algorithmics BONITO chip.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/device.h>
#include <sys/extent.h>
#include <sys/malloc.h>

#include <machine/autoconf.h>
#include <machine/bus.h>
#include <machine/cpu.h>
#include <mips64/mips_cpu.h>
#include <machine/intr.h>

#include <dev/pci/pcidevs.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <dev/pci/ppbreg.h>

#include <loongson/dev/bonitoreg.h>
#include <loongson/dev/bonitovar.h>
#include <loongson/dev/bonito_irq.h>

#include <uvm/uvm_extern.h>

#if 0
#define	BONITO_DEBUG
#endif

int	bonito_match(struct device *, void *, void *);
void	bonito_attach(struct device *, struct device *, void *);

const struct cfattach bonito_ca = {
	sizeof(struct bonito_softc), bonito_match, bonito_attach
};

struct cfdriver bonito_cd = {
	NULL, "bonito", DV_DULL
};

#define	wbflush()	mips_sync()

bus_addr_t	bonito_pa_to_device(paddr_t);
paddr_t		bonito_device_to_pa(bus_addr_t);

void	 bonito_intr_makemasks(void);
uint32_t bonito_intr_2e(uint32_t, struct trapframe *);
uint32_t bonito_intr_2f(uint32_t, struct trapframe *);
void	 bonito_intr_dispatch(uint64_t, int, struct trapframe *);

void	 bonito_attach_hook(struct device *, struct device *,
	    struct pcibus_attach_args *);
int	 bonito_bus_maxdevs(void *, int);
pcitag_t bonito_make_tag(void *, int, int, int);
void	 bonito_decompose_tag(void *, pcitag_t, int *, int *, int *);
int	 bonito_conf_size(void *, pcitag_t);
pcireg_t bonito_conf_read(void *, pcitag_t, int);
pcireg_t bonito_conf_read_internal(const struct bonito_config *, pcitag_t, int);
void	 bonito_conf_write(void *, pcitag_t, int, pcireg_t);
int	 bonito_pci_intr_map(struct pci_attach_args *, pci_intr_handle_t *);
const char *
	 bonito_pci_intr_string(void *, pci_intr_handle_t);
void	*bonito_pci_intr_establish(void *, pci_intr_handle_t, int,
	    int (*)(void *), void *, char *);
void	 bonito_pci_intr_disestablish(void *, void *);

int	 bonito_conf_addr(const struct bonito_config *, pcitag_t, int,
	    u_int32_t *, u_int32_t *);

void	 bonito_splx(int);

/*
 * Bonito interrupt handling declarations.
 * See <loongson/dev/bonito_irq.h> for details.
 */
struct intrhand *bonito_intrhand[BONITO_NINTS];
uint64_t bonito_intem;
uint64_t bonito_imask[NIPLS];

struct machine_bus_dma_tag bonito_bus_dma_tag = {
	._dmamap_create = _dmamap_create,
	._dmamap_destroy = _dmamap_destroy,
	._dmamap_load = _dmamap_load,
	._dmamap_load_mbuf = _dmamap_load_mbuf,
	._dmamap_load_uio = _dmamap_load_uio,
	._dmamap_load_raw = _dmamap_load_raw,
	._dmamap_load_buffer = _dmamap_load_buffer,
	._dmamap_unload = _dmamap_unload,
	._dmamap_sync = _dmamap_sync,

	._dmamem_alloc = _dmamem_alloc,
	._dmamem_free = _dmamem_free,
	._dmamem_map = _dmamem_map,
	._dmamem_unmap = _dmamem_unmap,
	._dmamem_mmap = _dmamem_mmap,

	._pa_to_device = bonito_pa_to_device,
	._device_to_pa = bonito_device_to_pa
};

int     bonito_io_map(bus_space_tag_t, bus_addr_t, bus_size_t, int,
	    bus_space_handle_t *);
int     bonito_mem_map(bus_space_tag_t, bus_addr_t, bus_size_t, int,
	    bus_space_handle_t *);

struct mips_bus_space bonito_pci_io_space_tag = {
	.bus_base = PHYS_TO_XKPHYS(BONITO_PCIIO_BASE, CCA_NC),
	._space_read_1 = generic_space_read_1,
	._space_write_1 = generic_space_write_1,
	._space_read_2 = generic_space_read_2,
	._space_write_2 = generic_space_write_2,
	._space_read_4 = generic_space_read_4,
	._space_write_4 = generic_space_write_4,
	._space_read_8 = generic_space_read_8,
	._space_write_8 = generic_space_write_8,
	._space_read_raw_2 = generic_space_read_raw_2,
	._space_write_raw_2 = generic_space_write_raw_2,
	._space_read_raw_4 = generic_space_read_raw_4,
	._space_write_raw_4 = generic_space_write_raw_4,
	._space_read_raw_8 = generic_space_read_raw_8,
	._space_write_raw_8 = generic_space_write_raw_8,
	._space_map = bonito_io_map,
	._space_unmap = generic_space_unmap,
	._space_subregion = generic_space_region,
	._space_vaddr = generic_space_vaddr,
	._space_mmap = generic_space_mmap
};

struct mips_bus_space bonito_pci_mem_space_tag = {
	.bus_base = PHYS_TO_XKPHYS(0, CCA_NC),
	._space_read_1 = generic_space_read_1,
	._space_write_1 = generic_space_write_1,
	._space_read_2 = generic_space_read_2,
	._space_write_2 = generic_space_write_2,
	._space_read_4 = generic_space_read_4,
	._space_write_4 = generic_space_write_4,
	._space_read_8 = generic_space_read_8,
	._space_write_8 = generic_space_write_8,
	._space_read_raw_2 = generic_space_read_raw_2,
	._space_write_raw_2 = generic_space_write_raw_2,
	._space_read_raw_4 = generic_space_read_raw_4,
	._space_write_raw_4 = generic_space_write_raw_4,
	._space_read_raw_8 = generic_space_read_raw_8,
	._space_write_raw_8 = generic_space_write_raw_8,
	._space_map = bonito_mem_map,
	._space_unmap = generic_space_unmap,
	._space_subregion = generic_space_region,
	._space_vaddr = generic_space_vaddr,
	._space_mmap = generic_space_mmap
};

int
bonito_match(struct device *parent, void *vcf, void *aux)
{
	struct mainbus_attach_args *maa = aux;

	if (loongson_ver >= 0x3a)
		return (0);

	if (strcmp(maa->maa_name, bonito_cd.cd_name) == 0)
		return (1);

	return (0);
}

void
bonito_attach(struct device *parent, struct device *self, void *aux)
{
	struct bonito_softc *sc = (struct bonito_softc *)self;
	struct pcibus_attach_args pba;
	pci_chipset_tag_t pc = &sc->sc_pc;
	const struct bonito_config *bc;
	uint32_t reg;

	/*
	 * Loongson 2F processors do not use a real Bonito64 chip but
	 * their own derivative, which is no longer 100% compatible.
	 * We need to make sure we never try to access an unimplemented
	 * register...
	 */
	if (loongson_ver >= 0x2f)
		sc->sc_compatible = 0;
	else
		sc->sc_compatible = 1;

	reg = PCI_REVISION(REGVAL(BONITO_PCI_REG(PCI_CLASS_REG)));
	if (sc->sc_compatible) {
		printf(": BONITO Memory and PCI controller, %s rev %d.%d\n",
		    BONITO_REV_FPGA(reg) ? "FPGA" : "ASIC",
		    BONITO_REV_MAJOR(reg), BONITO_REV_MINOR(reg));
	} else {
		printf(": memory and PCI-X controller, rev %d\n",
		    PCI_REVISION(REGVAL(BONITO_PCI_REG(PCI_CLASS_REG))));
	}

	bc = sys_platform->bonito_config;
	sc->sc_bonito = bc;
	SLIST_INIT(&sc->sc_hook);

#ifdef BONITO_DEBUG
	if (!sc->sc_compatible)
		printf("ISR4C: %08x\n", REGVAL(BONITO_PCI_REG(0x4c)));
	printf("PCIMAP: %08x\n", REGVAL(BONITO_PCIMAP));
	printf("MEMWIN: %08x.%08x - %08x.%08x\n",
	    REGVAL(BONITO_MEM_WIN_BASE_H), REGVAL(BONITO_MEM_WIN_BASE_L),
	    REGVAL(BONITO_MEM_WIN_MASK_H), REGVAL(BONITO_MEM_WIN_MASK_L));
	if (!sc->sc_compatible) {
		printf("HITSEL0: %08x.%08x\n",
		    REGVAL(LOONGSON_PCI_HIT0_SEL_H),
		    REGVAL(LOONGSON_PCI_HIT0_SEL_L));
		printf("HITSEL1: %08x.%08x\n",
		    REGVAL(LOONGSON_PCI_HIT1_SEL_H),
		    REGVAL(LOONGSON_PCI_HIT1_SEL_L));
		printf("HITSEL2: %08x.%08x\n",
		    REGVAL(LOONGSON_PCI_HIT2_SEL_H),
		    REGVAL(LOONGSON_PCI_HIT2_SEL_L));
	}
	printf("PCI BAR 0:%08x 1:%08x 2:%08x 3:%08x 4:%08x 5:%08x\n",
	    REGVAL(BONITO_PCI_REG(PCI_MAPREG_START + 0 * 4)),
	    REGVAL(BONITO_PCI_REG(PCI_MAPREG_START + 1 * 4)),
	    REGVAL(BONITO_PCI_REG(PCI_MAPREG_START + 2 * 4)),
	    REGVAL(BONITO_PCI_REG(PCI_MAPREG_START + 3 * 4)),
	    REGVAL(BONITO_PCI_REG(PCI_MAPREG_START + 4 * 4)),
	    REGVAL(BONITO_PCI_REG(PCI_MAPREG_START + 5 * 4)));
#endif

	/*
	 * Setup proper abitration.
	 */

	if (!sc->sc_compatible) {
		/*
		 * According to Linux, changing the value of this register
		 * ``avoids deadlock of PCI reading/writing lock operation''.
		 *
		 * Unfortunately, documentation for the Implementation
		 * Specific Registers (ISR40 to ISR5C) is only found in the
		 * chinese version of the Loongson 2F documentation.
		 *
		 * The particular bit we set here is ``mas_read_defer''.
		 */
		/* c2000001 -> d2000001 */
		REGVAL(BONITO_PCI_REG(0x4c)) |= 0x10000000;

		/* all pci devices may need to hold the bus */
		reg = REGVAL(LOONGSON_PXARB_CFG);
		reg &= ~LOONGSON_PXARB_RUDE_DEV_MSK;
		reg |= 0xfe << LOONGSON_PXARB_RUDE_DEV_SHFT;
		REGVAL(LOONGSON_PXARB_CFG) = reg;
		(void)REGVAL(LOONGSON_PXARB_CFG);
	}

	/*
	 * Setup interrupt handling.
	 */

	REGVAL(BONITO_GPIOIE) = bc->bc_gpioIE;
	REGVAL(BONITO_INTEDGE) = bc->bc_intEdge;
	if (sc->sc_compatible)
		REGVAL(BONITO_INTSTEER) = bc->bc_intSteer;
	REGVAL(BONITO_INTPOL) = bc->bc_intPol;

	REGVAL(BONITO_INTENCLR) = 0xffffffff;
	(void)REGVAL(BONITO_INTENCLR);
	
	if (sc->sc_compatible) {
		bonito_intem |= BONITO_INTRMASK_MASTERERR;
	}

	if (loongson_ver >= 0x2f)
		set_intr(INTPRI_BONITO, CR_INT_4, bonito_intr_2f);
	else
		set_intr(INTPRI_BONITO, CR_INT_0, bonito_intr_2e);
	register_splx_handler(bonito_splx);

	/*
	 * Attach PCI bus.
	 */

	pc->pc_conf_v = sc;
	pc->pc_attach_hook = bonito_attach_hook;
	pc->pc_bus_maxdevs = bonito_bus_maxdevs;
	pc->pc_make_tag = bonito_make_tag;
	pc->pc_decompose_tag = bonito_decompose_tag;
	pc->pc_conf_size = bonito_conf_size;
	pc->pc_conf_read = bonito_conf_read;
	pc->pc_conf_write = bonito_conf_write;

	pc->pc_intr_v = sc;
	pc->pc_intr_map = bonito_pci_intr_map;
	pc->pc_intr_string = bonito_pci_intr_string;
	pc->pc_intr_establish = bonito_pci_intr_establish;
	pc->pc_intr_disestablish = bonito_pci_intr_disestablish;

	bzero(&pba, sizeof pba);
	pba.pba_busname = "pci";
	pba.pba_iot = &bonito_pci_io_space_tag;
	pba.pba_memt = &bonito_pci_mem_space_tag;
	pba.pba_dmat = &bonito_bus_dma_tag;
	pba.pba_pc = pc;
	pba.pba_domain = pci_ndomains++;
	pba.pba_bus = 0;
#ifdef notyet
	pba.pba_ioex = bonito_get_resource_extent(pc, 1);
	pba.pba_memex = bonito_get_resource_extent(pc, 0);
#endif

	config_found(&sc->sc_dev, &pba, bonito_print);
}

bus_addr_t
bonito_pa_to_device(paddr_t pa)
{
	return pa ^ loongson_dma_base;
}

paddr_t
bonito_device_to_pa(bus_addr_t addr)
{
	return addr ^ loongson_dma_base;
}

int
bonito_print(void *aux, const char *pnp)
{
	struct pcibus_attach_args *pba = aux;

	if (pnp)
		printf("%s at %s", pba->pba_busname, pnp);
	printf(" bus %d", pba->pba_bus);

	return UNCONF;
}

/*
 * Bonito interrupt handling
 */

void *
bonito_intr_establish(int irq, int type, int level, int (*handler)(void *),
    void *arg, const char *name)
{
	struct intrhand **p, *q, *ih;
	int s;

#ifdef DIAGNOSTIC
	if (irq >= BONITO_NINTS || irq == BONITO_ISA_IRQ(2) || irq < 0)
		panic("bonito_intr_establish: illegal irq %d", irq);
#endif

	level &= ~IPL_MPSAFE;

	ih = malloc(sizeof *ih, M_DEVBUF, M_NOWAIT);
	if (ih == NULL)
		return NULL;

	ih->ih_next = NULL;
	ih->ih_fun = handler;
	ih->ih_arg = arg;
	ih->ih_level = level;
	ih->ih_irq = irq;
	evcount_attach(&ih->ih_count, name, &ih->ih_irq);

	s = splhigh();

	/*
	 * Figure out where to put the handler.
	 * This is O(N^2), but we want to preserve the order, and N is
	 * generally small.
	 */
	for (p = &bonito_intrhand[irq]; (q = *p) != NULL; p = &q->ih_next)
		;
	*p = ih;

	bonito_intem |= 1UL << irq;
	bonito_intr_makemasks();

	splx(s);	/* causes hw mask update */

	return (ih);
}

void
bonito_intr_disestablish(void *vih)
{
	struct intrhand *ih = (struct intrhand *)vih;
	struct intrhand **p, *q;
	int irq = ih->ih_irq;
	int s;

#ifdef DIAGNOSTIC
	if (irq >= BONITO_NINTS || irq == BONITO_ISA_IRQ(2) || irq < 0)
		panic("bonito_intr_disestablish: illegal irq %d", irq);
#endif

	s = splhigh();

	evcount_detach(&ih->ih_count);

	for (p = &bonito_intrhand[irq]; (q = *p) != NULL; p = &q->ih_next)
		if (q == ih)
			break;
#ifdef DIAGNOSTIC
	if (q == NULL)
		panic("bonito_intr_disestablish: never registered");
#endif
	*p = ih->ih_next;

	if (ih->ih_next == NULL && p == &bonito_intrhand[irq]) {
		bonito_intem &= ~(1UL << irq);
		bonito_intr_makemasks();
		/*
		 * No need to clear a bit in INTEN through INTCLR,
		 * splhigh() took care of disabling everything and
		 * splx() will not reenable this source after the
		 * mask update.
		 */
	}

	splx(s);

	free(ih, M_DEVBUF, sizeof *ih);
}

/*
 * Update interrupt masks. This is for designs without legacy PIC.
 */

void
bonito_splx(int newipl)
{
	struct cpu_info *ci = curcpu();

	/* Update masks to new ipl. Order highly important! */
	ci->ci_ipl = newipl;
	bonito_setintrmask(newipl);
	/* If we still have softints pending trigger processing. */
	if (ci->ci_softpending != 0 && newipl < IPL_SOFTINT)
		setsoftintr0();
}

void
bonito_setintrmask(int level)
{
	uint64_t active;
	uint32_t clear, set;
	register_t sr;

	active = bonito_intem & ~bonito_imask[level];
	/* be sure to mask high bits, there may be other interrupt sources */
	clear = BONITO_DIRECT_MASK(bonito_imask[level]);
	set = BONITO_DIRECT_MASK(active);

	sr = disableintr();

	if (clear != 0) {
		REGVAL(BONITO_INTENCLR) = clear;
		(void)REGVAL(BONITO_INTENCLR);
	}
	if (set != 0) {
		REGVAL(BONITO_INTENSET) = set;
		(void)REGVAL(BONITO_INTENSET);
	}

	setsr(sr);
}

/*
 * Recompute interrupt masks.
 */
void
bonito_intr_makemasks()
{
	int irq, level;
	struct intrhand *q;
	uint intrlevel[BONITO_NINTS];

	/* First, figure out which levels each IRQ uses. */
	for (irq = 0; irq < BONITO_NINTS; irq++) {
		uint levels = 0;
		for (q = bonito_intrhand[irq]; q != NULL; q = q->ih_next)
			levels |= 1 << q->ih_level;
		intrlevel[irq] = levels;
	}

	/*
	 * Then figure out which IRQs use each level.
	 * Note that we make sure never to overwrite imask[IPL_HIGH], in
	 * case an interrupt occurs during intr_disestablish() and causes
	 * an unfortunate splx() while we are here recomputing the masks.
	 */
	for (level = IPL_NONE; level < IPL_HIGH; level++) {
		uint64_t irqs = 0;
		for (irq = 0; irq < BONITO_NINTS; irq++)
			if (intrlevel[irq] & (1 << level))
				irqs |= 1UL << irq;
		bonito_imask[level] = irqs;
	}

	/*
	 * There are tty, network and disk drivers that use free() at interrupt
	 * time, so vm > (tty | net | bio).
	 *
	 * Enforce a hierarchy that gives slow devices a better chance at not
	 * dropping data.
	 */
	bonito_imask[IPL_NET] |= bonito_imask[IPL_BIO];
	bonito_imask[IPL_TTY] |= bonito_imask[IPL_NET];
	bonito_imask[IPL_VM] |= bonito_imask[IPL_TTY];
	bonito_imask[IPL_CLOCK] |= bonito_imask[IPL_VM];

	/*
	 * These are pseudo-levels.
	 */
	bonito_imask[IPL_NONE] = 0;
	bonito_imask[IPL_HIGH] = -1UL;
}

/*
 * Process native interrupts
 */

uint32_t
bonito_intr_2e(uint32_t hwpend, struct trapframe *frame)
{
	uint64_t imr, isr, mask;

	isr = REGVAL(BONITO_INTISR);

	/*
	 * According to Linux code, Bonito64 - at least on Loongson
	 * systems - triggers an interrupt during DMA, which is to be
	 * ignored. Smells like a chip errata to me.
	 */
	while (ISSET(isr, BONITO_INTRMASK_MASTERERR)) {
		delay(1);
		isr = REGVAL(BONITO_INTISR);
	}

	isr &= BONITO_INTRMASK_GPIN;
	imr = REGVAL(BONITO_INTEN);
	isr &= imr;
#ifdef DEBUG
	printf("pci interrupt: imr %04x isr %04x\n", imr, isr);
#endif
	if (isr == 0)
		return 0;	/* not for us */

	/*
	 * Mask all pending interrupts.
	 */
	REGVAL(BONITO_INTENCLR) = isr;
	(void)REGVAL(BONITO_INTENCLR);

	/*
	 * If interrupts are spl-masked, mask them and wait for splx()
	 * to reenable them when necessary.
	 */
	if ((mask = isr & bonito_imask[frame->ipl]) != 0) {
		isr &= ~mask;
		imr &= ~mask;
	}

	/*
	 * Now process allowed interrupts.
	 */
	if (isr != 0) {
		bonito_intr_dispatch(isr, 30, frame);

		/*
		 * Reenable interrupts which have been serviced.
		 */
		REGVAL(BONITO_INTENSET) = imr;
		(void)REGVAL(BONITO_INTENSET);
	}

	return hwpend;
}

uint32_t
bonito_intr_2f(uint32_t hwpend, struct trapframe *frame)
{
	uint64_t imr, isr, mask;

	isr = REGVAL(BONITO_INTISR) & LOONGSON_INTRMASK_LVL4;
	imr = REGVAL(BONITO_INTEN);
	isr &= imr;
#ifdef DEBUG
	printf("pci interrupt: imr %04x isr %04x\n", imr, isr);
#endif
	if (isr == 0)
		return 0;	/* not for us */

	/*
	 * Mask all pending interrupts.
	 */
	REGVAL(BONITO_INTENCLR) = isr;
	(void)REGVAL(BONITO_INTENCLR);

	/*
	 * If interrupts are spl-masked, mask them and wait for splx()
	 * to reenable them when necessary.
	 */
	if ((mask = isr & bonito_imask[frame->ipl]) != 0) {
		isr &= ~mask;
		imr &= ~mask;
	}

	/*
	 * Now process allowed interrupts.
	 */
	if (isr != 0) {
		bonito_intr_dispatch(isr,
		    LOONGSON_INTR_DRAM_PARERR /* skip non-pci interrupts */,
		    frame);

		/*
		 * Reenable interrupts which have been serviced.
		 */
		REGVAL(BONITO_INTENSET) = imr;
		(void)REGVAL(BONITO_INTENSET);
	}

	return hwpend;
}

void
bonito_intr_dispatch(uint64_t isr, int startbit, struct trapframe *frame)
{
	int lvl, bitno;
	uint64_t tmpisr, mask;
	struct intrhand *ih;
	int rc;

	/* Service higher level interrupts first */
	for (lvl = IPL_HIGH - 1; lvl != IPL_NONE; lvl--) {
		tmpisr = isr & (bonito_imask[lvl] ^ bonito_imask[lvl - 1]);
		if (tmpisr == 0)
			continue;
		for (bitno = startbit, mask = 1UL << bitno; mask != 0;
		    bitno--, mask >>= 1) {
			if ((tmpisr & mask) == 0)
				continue;

			rc = 0;
			for (ih = bonito_intrhand[bitno]; ih != NULL;
			    ih = ih->ih_next) {
				splraise(ih->ih_level);
				if ((*ih->ih_fun)(ih->ih_arg) != 0) {
					rc = 1;
					ih->ih_count.ec_count++;
				}
				curcpu()->ci_ipl = frame->ipl;
			}
			if (rc == 0) {
				printf("spurious interrupt %d\n", bitno);
#ifdef DEBUG
				printf("ISR %08x IMR %08x ipl %d mask %08x\n",
				    REGVAL(BONITO_INTISR), REGVAL(BONITO_INTEN),
				    frame->ipl, bonito_imask[frame->ipl]);
#ifdef DDB
				db_enter();
#endif
#endif
			}

			if ((isr ^= mask) == 0)
				return;
			if ((tmpisr ^= mask) == 0)
				break;
		}
	}
}

/*
 * various PCI helpers
 */

void
bonito_attach_hook(struct device *parent, struct device *self,
    struct pcibus_attach_args *pba)
{
	pci_chipset_tag_t pc = pba->pba_pc;
	struct bonito_softc *sc = pc->pc_conf_v;
	const struct bonito_config *bc = sc->sc_bonito;

	if (pba->pba_bus != 0)
		return;

	(*bc->bc_attach_hook)(pc);
}

/*
 * PCI configuration space access routines
 */

int
bonito_bus_maxdevs(void *v, int busno)
{
	struct bonito_softc *sc = v;
	const struct bonito_config *bc = sc->sc_bonito;

	return busno == 0 ? 32 - bc->bc_adbase : 32;
}

pcitag_t
bonito_make_tag(void *unused, int b, int d, int f)
{
	return (b << 16) | (d << 11) | (f << 8);
}

void
bonito_decompose_tag(void *unused, pcitag_t tag, int *bp, int *dp, int *fp)
{
	if (bp != NULL)
		*bp = (tag >> 16) & 0xff;
	if (dp != NULL)
		*dp = (tag >> 11) & 0x1f;
	if (fp != NULL)
		*fp = (tag >> 8) & 0x7;
}

int
bonito_conf_addr(const struct bonito_config *bc, pcitag_t tag, int offset,
    u_int32_t *cfgoff, u_int32_t *pcimap_cfg)
{
	int b, d, f;

	bonito_decompose_tag(NULL, tag, &b, &d, &f);

	if (b == 0) {
		d += bc->bc_adbase;
		if (d > 31)
			return 1;
		*cfgoff = (1 << d) | (f << 8) | offset;
		*pcimap_cfg = 0;
	} else {
		*cfgoff = tag | offset;
		*pcimap_cfg = BONITO_PCIMAPCFG_TYPE1;
	}

	return 0;
}

/* PCI Configuration Space access hook structure */
struct bonito_cfg_hook {
	SLIST_ENTRY(bonito_cfg_hook) next;
	int	(*read)(void *, pci_chipset_tag_t, pcitag_t, int, pcireg_t *);
	int	(*write)(void *, pci_chipset_tag_t, pcitag_t, int, pcireg_t);
	void	*cookie;
};

int
bonito_pci_hook(pci_chipset_tag_t pc, void *cookie,
    int (*r)(void *, pci_chipset_tag_t, pcitag_t, int, pcireg_t *),
    int (*w)(void *, pci_chipset_tag_t, pcitag_t, int, pcireg_t))
{
	struct bonito_softc *sc = pc->pc_conf_v;
	struct bonito_cfg_hook *bch;

	bch = malloc(sizeof *bch, M_DEVBUF, M_NOWAIT);
	if (bch == NULL)
		return ENOMEM;

	bch->read = r;
	bch->write = w;
	bch->cookie = cookie;
	SLIST_INSERT_HEAD(&sc->sc_hook, bch, next);
	return 0;
}

int
bonito_conf_size(void *v, pcitag_t tag)
{
	return PCI_CONFIG_SPACE_SIZE;
}

pcireg_t
bonito_conf_read(void *v, pcitag_t tag, int offset)
{
	struct bonito_softc *sc = v;
	struct bonito_cfg_hook *hook;
	pcireg_t data;

	SLIST_FOREACH(hook, &sc->sc_hook, next) {
		if (hook->read != NULL &&
		    (*hook->read)(hook->cookie, &sc->sc_pc, tag, offset,
		      &data) != 0)
			return data;
	}

	return bonito_conf_read_internal(sc->sc_bonito, tag, offset);
}

pcireg_t
bonito_conf_read_internal(const struct bonito_config *bc, pcitag_t tag,
    int offset)
{
	pcireg_t data;
	u_int32_t cfgoff, pcimap_cfg;
	register_t sr;
	uint64_t imr;

	if (bonito_conf_addr(bc, tag, offset, &cfgoff, &pcimap_cfg))
		return (pcireg_t)-1;

	sr = disableintr();
	imr = REGVAL(BONITO_INTEN);
	REGVAL(BONITO_INTENCLR) = 0xffffffff;
	(void)REGVAL(BONITO_INTENCLR);

	/* clear aborts */
	REGVAL(BONITO_PCI_REG(PCI_COMMAND_STATUS_REG)) |=
	    PCI_STATUS_MASTER_ABORT | PCI_STATUS_MASTER_TARGET_ABORT;

	/* high 16 bits of address go into PciMapCfg register */
	REGVAL(BONITO_PCIMAP_CFG) = (cfgoff >> 16) | pcimap_cfg;
	(void)REGVAL(BONITO_PCIMAP_CFG);
	wbflush();

	/* low 16 bits of address are offset into config space */
	data = REGVAL(BONITO_PCICFG_BASE + (cfgoff & 0xfffc));

	/* check for error */
	if (REGVAL(BONITO_PCI_REG(PCI_COMMAND_STATUS_REG)) &
	    (PCI_STATUS_MASTER_ABORT | PCI_STATUS_MASTER_TARGET_ABORT)) {
		REGVAL(BONITO_PCI_REG(PCI_COMMAND_STATUS_REG)) |=
		    PCI_STATUS_MASTER_ABORT | PCI_STATUS_MASTER_TARGET_ABORT;
		data = (pcireg_t) -1;
	}

	REGVAL(BONITO_INTENSET) = imr;
	(void)REGVAL(BONITO_INTENSET);
	setsr(sr);

	return data;
}

void
bonito_conf_write(void *v, pcitag_t tag, int offset, pcireg_t data)
{
	struct bonito_softc *sc = v;
	u_int32_t cfgoff, pcimap_cfg;
	struct bonito_cfg_hook *hook;
	register_t sr;
	uint64_t imr;

	SLIST_FOREACH(hook, &sc->sc_hook, next) {
		if (hook->write != NULL &&
		    (*hook->write)(hook->cookie, &sc->sc_pc, tag, offset,
		      data) != 0)
			return;
	}

	if (bonito_conf_addr(sc->sc_bonito, tag, offset, &cfgoff, &pcimap_cfg))
		panic("bonito_conf_write");

	sr = disableintr();
	imr = REGVAL(BONITO_INTEN);
	REGVAL(BONITO_INTENCLR) = 0xffffffff;
	(void)REGVAL(BONITO_INTENCLR);

	/* clear aborts */
	REGVAL(BONITO_PCI_REG(PCI_COMMAND_STATUS_REG)) |=
	    PCI_STATUS_MASTER_ABORT | PCI_STATUS_MASTER_TARGET_ABORT;

	/* high 16 bits of address go into PciMapCfg register */
	REGVAL(BONITO_PCIMAP_CFG) = (cfgoff >> 16) | pcimap_cfg;
	(void)REGVAL(BONITO_PCIMAP_CFG);
	wbflush();

	/* low 16 bits of address are offset into config space */
	REGVAL(BONITO_PCICFG_BASE + (cfgoff & 0xfffc)) = data;

	REGVAL(BONITO_INTENSET) = imr;
	(void)REGVAL(BONITO_INTENSET);
	setsr(sr);
}

/*
 * PCI Interrupt handling
 */

int
bonito_pci_intr_map(struct pci_attach_args *pa, pci_intr_handle_t *ihp)
{
	struct bonito_softc *sc = pa->pa_pc->pc_intr_v;
	const struct bonito_config *bc = sc->sc_bonito;
	int bus, dev, fn, pin;

	*ihp = (pci_intr_handle_t)-1;

	if (pa->pa_intrpin == 0)	/* no interrupt needed */
		return 1;

#ifdef DIAGNOSTIC
	if (pa->pa_intrpin > 4) {
		printf("%s: bad interrupt pin %d\n", __func__, pa->pa_intrpin);
		return 1;
	}
#endif

	pci_decompose_tag(pa->pa_pc, pa->pa_tag, &bus, &dev, &fn);
	if (pa->pa_bridgetag) {
		pin = PPB_INTERRUPT_SWIZZLE(pa->pa_rawintrpin, dev);
		*ihp = pa->pa_bridgeih[pin - 1];
	} else {
		if (bus == 0)
			*ihp = (*bc->bc_intr_map)(dev, fn, pa->pa_intrpin);

		if (*ihp == (pci_intr_handle_t)-1)
			return 1;
	}

	return 0;
}

const char *
bonito_pci_intr_string(void *cookie, pci_intr_handle_t ih)
{
	static char irqstr[1 + 12];

	if (BONITO_IRQ_IS_ISA(ih))
		snprintf(irqstr, sizeof irqstr, "isa irq %lu",
		    BONITO_IRQ_TO_ISA(ih));
	else
		snprintf(irqstr, sizeof irqstr, "irq %lu", ih);
	return irqstr;
}

void *
bonito_pci_intr_establish(void *cookie, pci_intr_handle_t ih, int level,
    int (*cb)(void *), void *cbarg, char *name)
{
	return bonito_intr_establish(ih, IST_LEVEL, level, cb, cbarg, name);
}

void
bonito_pci_intr_disestablish(void *cookie, void *ihp)
{
	bonito_intr_disestablish(ihp);
}

/*
 * bus_space mapping routines.
 */

/*
 * Legacy I/O access protection.
 * Since MI ISA code does not expect bus access to cause any failure when
 * accessing missing hardware, but only receive bogus data in return, we
 * force bus_space_map() to fail if there is no hardware there.
 */

int
bonito_io_map(bus_space_tag_t t, bus_addr_t offs, bus_size_t size, int flags,
    bus_space_handle_t *bshp)
{
	const struct legacy_io_range *r;
	bus_addr_t rend;

	if (offs < BONITO_PCIIO_LEGACY) {
		if ((r = sys_platform->legacy_io_ranges) == NULL)
			return ENXIO;

		rend = offs + size - 1;
		for (; r->start != 0; r++)
			if (offs >= r->start && rend <= r->end)
				break;

		if (r->end == 0)
			return ENXIO;
	}

	*bshp = t->bus_base + offs;
	return 0;
}

/*
 * PCI memory access.
 * Things are a bit complicated here, as we can either use one of the 64MB
 * windows in PCILO space (making sure ranges spanning multiple windows will
 * turn contiguous), or a direct access within the PCIHI space.
 * Note that, on 2F systems, only the PCIHI range for which CPU->PCI accesses
 * are enabled in the crossbar is usable.
 */

int
bonito_mem_map(bus_space_tag_t t, bus_addr_t offs, bus_size_t size, int flags,
    bus_space_handle_t *bshp)
{
	uint32_t pcimap;
	bus_addr_t pcilo_w[3];
	bus_addr_t ws, we, w;
	bus_addr_t end = offs + size - 1;
	int pcilo_window;

	pcimap = REGVAL(BONITO_PCIMAP);

	/*
	 * Try a PCIHI mapping first.
	 */

	if (loongson_ver >= 0x2f) {
		if (offs >= LS2F_PCIHI_BASE && end <= LS2F_PCIHI_TOP) {
			*bshp = t->bus_base + offs;
			return 0;
		}
	} else {
		/*
		 * Only try HI space if we do not have memory setup there.
		 */
		if (physmem <= atop(BONITO_PCILO_BASE)) {
			/* PCI1.5 */
			if (offs >= BONITO_PCIHI_BASE &&
			    end <= BONITO_PCIHI_TOP) {
				*bshp = t->bus_base + offs;
				return 0;
			}

			/* PCI2 */
			w = pcimap & BONITO_PCIMAP_PCIMAP_2 ? 0x80000000UL : 0;
			if (offs >= w && end < (w + 0x80000000UL)) {
				*bshp = t->bus_base + 0x80000000UL + (offs - w);
				return 0;
			}
		}
	}

	/*
	 * No luck, try a PCILO mapping.
	 */

	/*
	 * Decode PCIMAP, and figure out what PCILO mappings are
	 * possible.
	 */

	pcilo_w[0] = (pcimap & BONITO_PCIMAP_PCIMAP_LO0) >>
	    BONITO_PCIMAP_PCIMAP_LO0_SHIFT;
	pcilo_w[1] = (pcimap & BONITO_PCIMAP_PCIMAP_LO1) >>
	    BONITO_PCIMAP_PCIMAP_LO1_SHIFT;
	pcilo_w[2] = (pcimap & BONITO_PCIMAP_PCIMAP_LO2) >>
	    BONITO_PCIMAP_PCIMAP_LO2_SHIFT;

	/*
	 * Check if the 64MB areas we want to span are all available as
	 * contiguous PCILO mappings.
	 */

	ws = offs >> 26;
	we = end >> 26;

	pcilo_window = -1;
	if (ws == pcilo_w[0])
		pcilo_window = 0;
	else if (ws == pcilo_w[1])
		pcilo_window = 1;
	else if (ws == pcilo_w[2])
		pcilo_window = 2;

	if (pcilo_window >= 0) {
		/* contiguous area test */
		for (w = ws + 1; w <= we; w++) {
			if (pcilo_window + (w - ws) > 2 ||
			    w != pcilo_w[pcilo_window + (w - ws)]) {
				pcilo_window = -1;
				break;
			}
		}
	}

	if (pcilo_window >= 0) {
		*bshp = t->bus_base + BONITO_PCILO_BASE +
		    BONITO_PCIMAP_WINBASE(pcilo_window) +
		    BONITO_PCIMAP_WINOFFSET(offs);
		return 0;
	}

	return EINVAL;
}

/*
 * PCI resource handling
 */

struct extent *
bonito_get_resource_extent(pci_chipset_tag_t pc, int io)
{
	struct bonito_softc *sc = pc->pc_conf_v;
	struct extent *ex;
	char *exname;
	size_t exnamesz;
	uint32_t reg;
	int errors;

	exnamesz = 1 + 16 + 4;
	exname = (char *)malloc(exnamesz, M_DEVBUF, M_NOWAIT);
	if (exname == NULL)
		return NULL;
	snprintf(exname, exnamesz, "%s%s", sc->sc_dev.dv_xname,
	    io ? "_io" : "_mem");

	ex = extent_create(exname, 0, 0xffffffff, M_DEVBUF, NULL, 0,
	    EX_NOWAIT | EX_FILLED);
	if (ex == NULL)
		goto out;

	errors = 0;
	if (io) {
		/*
		 * Reserve the low 16KB of I/O space to the legacy hardware,
		 * if any.
		 */
		if (extent_free(ex, BONITO_PCIIO_LEGACY, BONITO_PCIIO_SIZE,
		    EX_NOWAIT) != 0)
			errors++;
	} else {
		reg = REGVAL(BONITO_PCIMAP);
		if (extent_free(ex,
		    BONITO_PCIMAP_WINBASE((reg & BONITO_PCIMAP_PCIMAP_LO0) >>
		      BONITO_PCIMAP_PCIMAP_LO0_SHIFT),
		    BONITO_PCIMAP_WINSIZE, EX_NOWAIT) != 0)
			errors++;
		if (extent_free(ex,
		    BONITO_PCIMAP_WINBASE((reg & BONITO_PCIMAP_PCIMAP_LO1) >>
		      BONITO_PCIMAP_PCIMAP_LO1_SHIFT),
		    BONITO_PCIMAP_WINSIZE, EX_NOWAIT) != 0)
			errors++;
		if (extent_free(ex,
		    BONITO_PCIMAP_WINBASE((reg & BONITO_PCIMAP_PCIMAP_LO2) >>
		      BONITO_PCIMAP_PCIMAP_LO2_SHIFT),
		    BONITO_PCIMAP_WINSIZE, EX_NOWAIT) != 0)
			errors++;

		if (sc->sc_compatible) {
			/* XXX make PCIMAP_HI available if PCIMAP_2 set */
		}
	}

	if (errors != 0) {
		extent_destroy(ex);
		ex = NULL;
	}

#ifdef BONITO_DEBUG
	extent_print(ex);
#endif

out:
	free(exname, M_DEVBUF, exnamesz);

	return ex;
}

/*
 * Functions used during early system configuration (before bonito attaches).
 */

pcitag_t bonito_make_tag_early(int, int, int);
pcireg_t bonito_conf_read_early(pcitag_t, int);

pcitag_t
bonito_make_tag_early(int b, int d, int f)
{
	return bonito_make_tag(NULL, b, d, f);
}

pcireg_t
bonito_conf_read_early(pcitag_t tag, int reg)
{
	return bonito_conf_read_internal(sys_platform->bonito_config, tag, reg);
}

void
bonito_early_setup()
{
	pci_make_tag_early = bonito_make_tag_early;
	pci_conf_read_early = bonito_conf_read_early;

	early_mem_t = &bonito_pci_mem_space_tag;
	early_io_t = &bonito_pci_io_space_tag;
}