diff options
Diffstat (limited to 'sys/dev/pci/drm')
| -rw-r--r-- | sys/dev/pci/drm/drmP.h | 83 | ||||
| -rw-r--r-- | sys/dev/pci/drm/drm_drv.c | 465 | ||||
| -rw-r--r-- | sys/dev/pci/drm/i915_dma.c | 213 | ||||
| -rw-r--r-- | sys/dev/pci/drm/i915_drv.c | 4471 | ||||
| -rw-r--r-- | sys/dev/pci/drm/i915_drv.h | 272 | ||||
| -rw-r--r-- | sys/dev/pci/drm/i915_irq.c | 115 |
6 files changed, 5424 insertions, 195 deletions
diff --git a/sys/dev/pci/drm/drmP.h b/sys/dev/pci/drm/drmP.h index cedc82d2e0e..5d8f6046216 100644 --- a/sys/dev/pci/drm/drmP.h +++ b/sys/dev/pci/drm/drmP.h @@ -40,6 +40,7 @@ #include <sys/param.h> #include <sys/queue.h> #include <sys/malloc.h> +#include <sys/pool.h> #include <sys/kernel.h> #include <sys/systm.h> #include <sys/conf.h> @@ -99,7 +100,7 @@ #define DRM_SUSER(p) (suser(p, p->p_acflag) == 0) #define DRM_MTRR_WC MDF_WRITECOMBINE -#define PAGE_ALIGN(addr) (((addr) + PAGE_SIZE - 1) & PAGE_MASK) +#define PAGE_ALIGN(addr) (((addr) + PAGE_MASK) & ~PAGE_MASK) extern struct cfdriver drm_cd; @@ -215,14 +216,17 @@ struct drm_buf_entry { }; struct drm_file { - SPLAY_ENTRY(drm_file) link; - int authenticated; - unsigned long ioctl_count; - dev_t kdev; - drm_magic_t magic; - int flags; - int master; - int minor; + SPLAY_HEAD(drm_obj_tree, drm_handle) obj_tree; + struct mutex table_lock; + SPLAY_ENTRY(drm_file) link; + int authenticated; + unsigned long ioctl_count; + dev_t kdev; + drm_magic_t magic; + int flags; + int master; + int minor; + u_int obj_id; /*next gem id*/ }; struct drm_lock_data { @@ -349,6 +353,28 @@ struct drm_ati_pcigart_info { int gart_reg_if; }; +struct drm_obj { + struct uvm_object uobj; + SPLAY_ENTRY(drm_obj) entry; + struct drm_device *dev; + struct uvm_object *uao; + + size_t size; + int name; + int handlecount; + uint32_t read_domains; + uint32_t write_domain; + + uint32_t pending_read_domains; + uint32_t pending_write_domain; +}; + +struct drm_handle { + SPLAY_ENTRY(drm_handle) entry; + struct drm_obj *obj; + uint32_t handle; +}; + struct drm_driver_info { int (*firstopen)(struct drm_device *); int (*open)(struct drm_device *, struct drm_file *); @@ -367,7 +393,16 @@ struct drm_driver_info { u_int32_t (*get_vblank_counter)(struct drm_device *, int); int (*enable_vblank)(struct drm_device *, int); void (*disable_vblank)(struct drm_device *, int); - + /* + * driver-specific constructor for gem objects to set up private data. + * returns 0 on success. + */ + int (*gem_init_object)(struct drm_obj *); + void (*gem_free_object)(struct drm_obj *); + int (*gem_fault)(struct drm_obj *, struct uvm_faultinfo *, off_t, + vaddr_t, vm_page_t *, int, int, vm_prot_t, int); + + size_t gem_size; size_t buf_priv_size; size_t file_priv_size; @@ -385,6 +420,7 @@ struct drm_driver_info { #define DRIVER_PCI_DMA 0x10 #define DRIVER_SG 0x20 #define DRIVER_IRQ 0x40 +#define DRIVER_GEM 0x80 u_int flags; }; @@ -438,6 +474,21 @@ struct drm_device { atomic_t *ctx_bitmap; void *dev_private; struct drm_local_map *agp_buffer_map; + + /* GEM info */ + struct mutex obj_name_lock; + atomic_t obj_count; + u_int obj_name; + atomic_t obj_memory; + atomic_t pin_count; + atomic_t pin_memory; + atomic_t gtt_count; + atomic_t gtt_memory; + uint32_t gtt_total; + uint32_t invalidate_domains; + uint32_t flush_domains; + SPLAY_HEAD(drm_name_tree, drm_obj) name_tree; + struct pool objpl; }; struct drm_attach_args { @@ -588,5 +639,17 @@ int drm_agp_bind_ioctl(struct drm_device *, void *, struct drm_file *); int drm_sg_alloc_ioctl(struct drm_device *, void *, struct drm_file *); int drm_sg_free(struct drm_device *, void *, struct drm_file *); +struct drm_obj *drm_gem_object_alloc(struct drm_device *, size_t); +void drm_gem_object_reference(struct drm_obj *); +void drm_gem_object_unreference(struct drm_obj *); +int drm_handle_create(struct drm_file *, struct drm_obj *, int *); +struct drm_obj *drm_gem_object_lookup(struct drm_device *, + struct drm_file *, int ); +int drm_gem_close_ioctl(struct drm_device *, void *, struct drm_file *); +int drm_gem_flink_ioctl(struct drm_device *, void *, struct drm_file *); +int drm_gem_open_ioctl(struct drm_device *, void *, struct drm_file *); +int drm_gem_load_uao(bus_dma_tag_t, bus_dmamap_t, struct uvm_object *, + bus_size_t, int, bus_dma_segment_t **); + #endif /* __KERNEL__ */ #endif /* _DRM_P_H_ */ diff --git a/sys/dev/pci/drm/drm_drv.c b/sys/dev/pci/drm/drm_drv.c index 88a5e1daadf..4dfd6895ec6 100644 --- a/sys/dev/pci/drm/drm_drv.c +++ b/sys/dev/pci/drm/drm_drv.c @@ -1,4 +1,6 @@ /*- + * Copyright 2007-2009 Owain G. Ainsworth <oga@openbsd.org> + * Copyright © 2008 Intel Corporation * Copyright 2003 Eric Anholt * Copyright 1999, 2000 Precision Insight, Inc., Cedar Park, Texas. * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California. @@ -27,6 +29,8 @@ * Rickard E. (Rik) Faith <faith@valinux.com> * Daryll Strauss <daryll@valinux.com> * Gareth Hughes <gareth@valinux.com> + * Eric Anholt <eric@anholt.net> + * Owain Ainsworth <oga@openbsd.org> * */ @@ -35,7 +39,11 @@ * open/close, and ioctl dispatch. */ +#include <sys/param.h> #include <sys/limits.h> +#include <sys/systm.h> +#include <uvm/uvm_extern.h> + #include <sys/ttycom.h> /* for TIOCSGRP */ #include "drmP.h" @@ -64,6 +72,21 @@ int drm_authmagic(struct drm_device *, void *, struct drm_file *); int drm_file_cmp(struct drm_file *, struct drm_file *); SPLAY_PROTOTYPE(drm_file_tree, drm_file, link, drm_file_cmp); +/* functions used by the per-open handle code to grab references to object */ +void drm_handle_ref(struct drm_obj *); +void drm_handle_unref(struct drm_obj *); + +int drm_handle_cmp(struct drm_handle *, struct drm_handle *); +int drm_name_cmp(struct drm_obj *, struct drm_obj *); +void drm_unref(struct uvm_object *); +void drm_ref(struct uvm_object *); +int drm_fault(struct uvm_faultinfo *, vaddr_t, vm_page_t *, int, int, + vm_fault_t, vm_prot_t, int); +boolean_t drm_flush(struct uvm_object *, voff_t, voff_t, int); + +SPLAY_PROTOTYPE(drm_obj_tree, drm_handle, entry, drm_handle_cmp); +SPLAY_PROTOTYPE(drm_name_tree, drm_obj, entry, drm_name_cmp); + /* * attach drm to a pci-based driver. * @@ -180,6 +203,16 @@ drm_attach(struct device *parent, struct device *self, void *aux) printf(": couldn't allocate memory for context bitmap.\n"); goto error; } + + if (dev->driver->flags & DRIVER_GEM) { + mtx_init(&dev->obj_name_lock, IPL_NONE); + SPLAY_INIT(&dev->name_tree); + KASSERT(dev->driver->gem_size >= sizeof(struct drm_obj)); + /* XXX unique name */ + pool_init(&dev->objpl, dev->driver->gem_size, 0, 0, 0, + "drmobjpl", &pool_allocator_nointr); + } + printf("\n"); return; @@ -386,6 +419,11 @@ drmopen(dev_t kdev, int flags, int fmt, struct proc *p) /* for compatibility root is always authenticated */ file_priv->authenticated = DRM_SUSER(p); + if (dev->driver->flags & DRIVER_GEM) { + SPLAY_INIT(&file_priv->obj_tree); + mtx_init(&file_priv->table_lock, IPL_NONE); + } + if (dev->driver->open) { ret = dev->driver->open(dev, file_priv); if (ret != 0) { @@ -485,9 +523,20 @@ drmclose(dev_t kdev, int flags, int fmt, struct proc *p) !dev->driver->reclaim_buffers_locked) drm_reclaim_buffers(dev, file_priv); + DRM_LOCK(); + if (dev->driver->flags & DRIVER_GEM) { + struct drm_handle *han; + mtx_enter(&file_priv->table_lock); + while ((han = SPLAY_ROOT(&file_priv->obj_tree)) != NULL) { + SPLAY_REMOVE(drm_obj_tree, &file_priv->obj_tree, han); + drm_handle_unref(han->obj); + drm_free(han); + } + mtx_leave(&file_priv->table_lock); + } + dev->buf_pgid = 0; - DRM_LOCK(); SPLAY_REMOVE(drm_file_tree, &dev->files, file_priv); drm_free(file_priv); @@ -550,6 +599,8 @@ drmioctl(dev_t kdev, u_long cmd, caddr_t data, int flags, return (drm_wait_vblank(dev, data, file_priv)); case DRM_IOCTL_MODESET_CTL: return (drm_modeset_ctl(dev, data, file_priv)); + case DRM_IOCTL_GEM_CLOSE: + return (drm_gem_close_ioctl(dev, data, file_priv)); /* removed */ case DRM_IOCTL_GET_MAP: @@ -596,6 +647,11 @@ drmioctl(dev_t kdev, u_long cmd, caddr_t data, int flags, return (drm_dma(dev, data, file_priv)); case DRM_IOCTL_AGP_INFO: return (drm_agp_info_ioctl(dev, data, file_priv)); + case DRM_IOCTL_GEM_FLINK: + return (drm_gem_flink_ioctl(dev, data, file_priv)); + case DRM_IOCTL_GEM_OPEN: + return (drm_gem_open_ioctl(dev, data, file_priv)); + } } @@ -936,7 +992,6 @@ drm_getmagic(struct drm_device *dev, void *data, struct drm_file *file_priv) } DRM_DEBUG("%u\n", auth->magic); - return (0); } @@ -968,3 +1023,409 @@ drm_authmagic(struct drm_device *dev, void *data, struct drm_file *file_priv) return (ret); } + +struct uvm_pagerops drm_pgops = { + NULL, + drm_ref, + drm_unref, + drm_fault, + drm_flush, +}; + +void +drm_ref(struct uvm_object *uobj) +{ + simple_lock(&uobj->vmobjlock); + uobj->uo_refs++; + simple_unlock(&uobj->vmobjlock); +} + +void +drm_unref(struct uvm_object *uobj) +{ + struct drm_obj *obj = (struct drm_obj *)uobj; + struct drm_device *dev = obj->dev; + + simple_lock(&uobj->vmobjlock); + if (--uobj->uo_refs > 0) { + simple_unlock(&uobj->vmobjlock); + return; + } + + if (dev->driver->gem_free_object != NULL) + dev->driver->gem_free_object(obj); + + uao_detach(obj->uao); + + atomic_dec(&dev->obj_count); + atomic_sub(obj->size, &dev->obj_memory); + simple_unlock(&uobj->vmobjlock); + pool_put(&dev->objpl, obj); +} + + +boolean_t +drm_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags) +{ + return (TRUE); +} + + +int +drm_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr, vm_page_t *pps, + int npages, int centeridx, vm_fault_t fault_type, + vm_prot_t access_type, int flags) +{ + struct vm_map_entry *entry = ufi->entry; + struct uvm_object *uobj = entry->object.uvm_obj; + struct drm_obj *obj = (struct drm_obj *)uobj; + struct drm_device *dev = obj->dev; + int ret; + UVMHIST_FUNC("udv_fault"); UVMHIST_CALLED(maphist); + UVMHIST_LOG(maphist," flags=%ld", flags,0,0,0); + + /* + * we do not allow device mappings to be mapped copy-on-write + * so we kill any attempt to do so here. + */ + + if (UVM_ET_ISCOPYONWRITE(entry)) { + UVMHIST_LOG(maphist, "<- failed -- COW entry (etype=0x%lx)", + entry->etype, 0,0,0); + uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, uobj, NULL); + return(VM_PAGER_ERROR); + } + + /* Call down into driver to do the magic */ + ret = dev->driver->gem_fault(obj, ufi, entry->offset + (vaddr - + entry->start), vaddr, pps, npages, centeridx, + access_type, flags); + + uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, uobj, NULL); + pmap_update(ufi->orig_map->pmap); + if (ret != VM_PAGER_OK) + uvm_wait("drm_fault"); + return (ret); +} + +/* + * Code to support memory managers based on the GEM (Graphics + * Execution Manager) api. + */ +struct drm_obj * +drm_gem_object_alloc(struct drm_device *dev, size_t size) +{ + struct drm_obj *obj; + + KASSERT((size & (PAGE_SIZE -1)) == 0); + + if ((obj = pool_get(&dev->objpl, PR_WAITOK | PR_ZERO)) == NULL) + return (NULL); + + obj->dev = dev; + + /* uao create can't fail in the 0 case, it just sleeps */ + obj->uao = uao_create(size, 0); + obj->size = size; + simple_lock_init(&obj->uobj.vmobjlock); + obj->uobj.pgops = &drm_pgops; + RB_INIT(&obj->uobj.memt); + obj->uobj.uo_npages = 0; + obj->uobj.uo_refs = 1; + + if (dev->driver->gem_init_object != NULL && + dev->driver->gem_init_object(obj) != 0) { + uao_detach(obj->uao); + pool_put(&dev->objpl, obj); + return (NULL); + } + atomic_inc(&dev->obj_count); + atomic_add(obj->size, &dev->obj_memory); + return (obj); +} + +int +drm_handle_create(struct drm_file *file_priv, struct drm_obj *obj, + int *handlep) +{ + struct drm_handle *han; + + if ((han = drm_calloc(1, sizeof(*han))) == NULL) + return (ENOMEM); + + han->obj = obj; + mtx_enter(&file_priv->table_lock); +again: + *handlep = han->handle = ++file_priv->obj_id; + /* + * Make sure we have no duplicates. this'll hurt once we wrap, 0 is + * reserved. + */ + if (han->handle == 0 || SPLAY_INSERT(drm_obj_tree, + &file_priv->obj_tree, han)) + goto again; + mtx_leave(&file_priv->table_lock); + + drm_handle_ref(obj); + return (0); +} + +struct drm_obj * +drm_gem_object_lookup(struct drm_device *dev, struct drm_file *file_priv, + int handle) +{ + struct drm_obj *obj; + struct drm_handle *han, search; + + search.handle = handle; + + mtx_enter(&file_priv->table_lock); + han = SPLAY_FIND(drm_obj_tree, &file_priv->obj_tree, &search); + if (han == NULL) { + mtx_leave(&file_priv->table_lock); + return (NULL); + } + + obj = han->obj; + drm_gem_object_reference(obj); + mtx_leave(&file_priv->table_lock); + + return (obj); +} + +int +drm_gem_close_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_gem_close *args = data; + struct drm_handle *han, find; + struct drm_obj *obj; + + if ((dev->driver->flags & DRIVER_GEM) == 0) + return (ENODEV); + + find.handle = args->handle; + mtx_enter(&file_priv->table_lock); + han = SPLAY_FIND(drm_obj_tree, &file_priv->obj_tree, &find); + if (han == NULL) { + mtx_leave(&file_priv->table_lock); + return (EINVAL); + } + + obj = han->obj; + SPLAY_REMOVE(drm_obj_tree, &file_priv->obj_tree, han); + mtx_leave(&file_priv->table_lock); + + drm_free(han); + + DRM_LOCK(); + drm_handle_unref(obj); + DRM_UNLOCK(); + + return (0); +} + +int +drm_gem_flink_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_gem_flink *args = data; + struct drm_obj *obj; + + if (!dev->driver->flags & DRIVER_GEM) + return (ENODEV); + + obj = drm_gem_object_lookup(dev, file_priv, args->handle); + if (obj == NULL) + return (EBADF); + + mtx_enter(&dev->obj_name_lock); + if (!obj->name) { +again: + obj->name = ++dev->obj_name; + /* 0 is reserved, make sure we don't clash. */ + if (obj->name == 0 || SPLAY_INSERT(drm_name_tree, + &dev->name_tree, obj)) + goto again; + /* name holds a reference to the object */ + drm_gem_object_reference(obj); + } + mtx_leave(&dev->obj_name_lock); + + args->name = (uint64_t)obj->name; + + DRM_LOCK(); + drm_gem_object_unreference(obj); + DRM_UNLOCK(); + + return (0); +} + +int +drm_gem_open_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_gem_open *args = data; + struct drm_obj *obj, search; + int ret, handle; + + if (!dev->driver->flags & DRIVER_GEM) + return (ENODEV); + + search.name = args->name; + mtx_enter(&dev->obj_name_lock); + obj = SPLAY_FIND(drm_name_tree, &dev->name_tree, &search); + if (obj != NULL) + drm_gem_object_reference(obj); + mtx_leave(&dev->obj_name_lock); + if (obj == NULL) + return (ENOENT); + + ret = drm_handle_create(file_priv, obj, &handle); + /* handle has a reference, drop ours. */ + DRM_LOCK(); + drm_gem_object_unreference(obj); + DRM_UNLOCK(); + if (ret) { + return (ret); + } + + args->handle = handle; + args->size = obj->size; + + return (0); +} + +void +drm_gem_object_reference(struct drm_obj *obj) +{ + drm_ref(&obj->uobj); +} + +void +drm_gem_object_unreference(struct drm_obj *obj) +{ + drm_unref(&obj->uobj); +} + + +/* + * grab a reference for a per-open handle. + * The object contains a handlecount too because if all handles disappear we + * need to also remove the global name (names initially are per open unless the + * flink ioctl is called. + */ +void +drm_handle_ref(struct drm_obj *obj) +{ + obj->handlecount++; + drm_gem_object_reference(obj); +} + +/* + * Remove the reference owned by a per-open handle. If we're the last one, + * remove the reference from flink, too. + */ +void +drm_handle_unref(struct drm_obj *obj) +{ + /* do this first in case this is the last reference */ + if (--obj->handlecount == 0) { + struct drm_device *dev = obj->dev; + + mtx_enter(&dev->obj_name_lock); + if (obj->name) { + SPLAY_REMOVE(drm_name_tree, &dev->name_tree, obj); + obj->name = 0; + mtx_leave(&dev->obj_name_lock); + /* name held a reference to object */ + drm_gem_object_unreference(obj); + } else { + mtx_leave(&dev->obj_name_lock); + } + } + drm_gem_object_unreference(obj); +} + +/* + * Helper function to load a uvm anonymous object into a dmamap, to be used + * for binding to a translation-table style sg mechanism (e.g. agp, or intel + * gtt). + * + * For now we ignore maxsegsz. + */ +int +drm_gem_load_uao(bus_dma_tag_t dmat, bus_dmamap_t map, struct uvm_object *uao, + bus_size_t size, int flags, bus_dma_segment_t **segp) +{ + bus_dma_segment_t *segs; + struct vm_page *pg; + struct pglist plist; + u_long npages = size >> PAGE_SHIFT, i = 0; + int ret; + + TAILQ_INIT(&plist); + + /* + * This is really quite ugly, but nothing else would need + * bus_dmamap_load_uao() yet. + */ + segs = malloc(npages * sizeof(*segs), M_DRM, M_WAITOK | M_ZERO); + if (segs == NULL) + return (ENOMEM); + + /* This may sleep, no choice in the matter */ + if (uvm_objwire(uao, 0, size, &plist) != 0) { + ret = ENOMEM; + goto free; + } + + TAILQ_FOREACH(pg, &plist, pageq) { + paddr_t pa = VM_PAGE_TO_PHYS(pg); + + if (i > 0 && pa == (segs[i - 1].ds_addr + + segs[i - 1].ds_len)) { + /* contiguous, yay */ + segs[i - 1].ds_len += PAGE_SIZE; + continue; + } + segs[i].ds_addr = pa; + segs[i].ds_len = PAGE_SIZE; + if (i++ > npages) + break; + } + /* this should be impossible */ + if (pg != TAILQ_END(&pageq)) { + ret = EINVAL; + goto unwire; + } + + if ((ret = bus_dmamap_load_raw(dmat, map, segs, i, size, flags)) != 0) + goto unwire; + + *segp = segs; + + return (0); + +unwire: + uvm_objunwire(uao, 0, size); +free: + free(segs, M_DRM); + return (ret); +} + +int +drm_handle_cmp(struct drm_handle *a, struct drm_handle *b) +{ + return (a->handle < b->handle ? -1 : a->handle > b->handle); +} + +int +drm_name_cmp(struct drm_obj *a, struct drm_obj *b) +{ + return (a->name < b->name ? -1 : a->name > b->name); +} + +SPLAY_GENERATE(drm_obj_tree, drm_handle, entry, drm_handle_cmp); + +SPLAY_GENERATE(drm_name_tree, drm_obj, entry, drm_name_cmp); diff --git a/sys/dev/pci/drm/i915_dma.c b/sys/dev/pci/drm/i915_dma.c index 13c881dc7dc..a221dfb4768 100644 --- a/sys/dev/pci/drm/i915_dma.c +++ b/sys/dev/pci/drm/i915_dma.c @@ -31,6 +31,11 @@ #include "i915_drm.h" #include "i915_drv.h" +int i915_dispatch_batchbuffer(struct drm_device *, + drm_i915_batchbuffer_t *, struct drm_clip_rect *); +int i915_dispatch_cmdbuffer(struct drm_device *, + drm_i915_cmdbuffer_t *, struct drm_clip_rect *); + /** * Sets up the hardware status page for devices that need a physical address * in the register. @@ -83,15 +88,14 @@ i915_dma_cleanup(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; - if (dev_priv->ring.bsh != NULL) { + if (dev_priv->ring.ring_obj == NULL && dev_priv->ring.bsh != NULL) { bus_space_unmap(dev_priv->bst, dev_priv->ring.bsh, dev_priv->ring.size); - dev_priv->ring.bsh = NULL; - dev_priv->ring.size = 0; + memset(&dev_priv->ring, 0, sizeof(dev_priv->ring)); } /* Clear the HWS virtual address at teardown */ - if (I915_NEED_GFX_HWS(dev_priv)) + if (dev_priv->hws_obj == NULL && I915_NEED_GFX_HWS(dev_priv)) i915_free_hws(dev_priv, dev->dmat); return 0; @@ -111,20 +115,27 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init) if (init->sarea_priv_offset) dev_priv->sarea_priv = (drm_i915_sarea_t *) - ((u8 *) dev_priv->sarea->handle + - init->sarea_priv_offset); + ((u8 *) dev_priv->sarea->handle + init->sarea_priv_offset); else { /* No sarea_priv for you! */ dev_priv->sarea_priv = NULL; } - dev_priv->ring.size = init->ring_size; + if (init->ring_size != 0) { + if (dev_priv->ring.ring_obj != NULL) { + i915_dma_cleanup(dev); + DRM_ERROR("Client tried to initialize ringbuffer in " + "GEM mode\n"); + return (-EINVAL); + } + dev_priv->ring.size = init->ring_size; - if ((ret = bus_space_map(dev_priv->bst, init->ring_start, - init->ring_size, 0, &dev_priv->ring.bsh)) != 0) { - DRM_INFO("can't map ringbuffer\n"); - i915_dma_cleanup(dev); - return (ret); + if ((ret = bus_space_map(dev_priv->bst, init->ring_start, + init->ring_size, 0, &dev_priv->ring.bsh)) != 0) { + DRM_INFO("can't map ringbuffer\n"); + i915_dma_cleanup(dev); + return (ret); + } } /* Allow hardware batchbuffers unless told otherwise. @@ -299,43 +310,35 @@ static int i915_emit_cmds(struct drm_device *dev, int __user *buffer, return 0; } -static int -i915_emit_box(struct drm_device * dev, struct drm_clip_rect *boxes, - int i, int DR1, int DR4) + +/* + * Emit a box for a cliprect. cliprect must already have been copied in and + * sanity checked (the reason for this is so that everything can be checked + * before any gpu state is modified. + */ +void +i915_emit_box(struct drm_device * dev, struct drm_clip_rect *box, + int DR1, int DR4) { drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_clip_rect box; - - if (DRM_COPY_FROM_USER(&box, &boxes[i], sizeof(box))) { - return EFAULT; - } - - if (box.y2 <= box.y1 || box.x2 <= box.x1 || box.y2 <= 0 || - box.x2 <= 0) { - DRM_ERROR("Bad box %d,%d..%d,%d\n", - box.x1, box.y1, box.x2, box.y2); - return EINVAL; - } if (IS_I965G(dev_priv)) { BEGIN_LP_RING(4); OUT_RING(GFX_OP_DRAWRECT_INFO_I965); - OUT_RING((box.x1 & 0xffff) | (box.y1 << 16)); - OUT_RING(((box.x2 - 1) & 0xffff) | ((box.y2 - 1) << 16)); + OUT_RING((box->x1 & 0xffff) | (box->y1 << 16)); + OUT_RING(((box->x2 - 1) & 0xffff) | ((box->y2 - 1) << 16)); OUT_RING(DR4); ADVANCE_LP_RING(); } else { BEGIN_LP_RING(6); OUT_RING(GFX_OP_DRAWRECT_INFO); OUT_RING(DR1); - OUT_RING((box.x1 & 0xffff) | (box.y1 << 16)); - OUT_RING(((box.x2 - 1) & 0xffff) | ((box.y2 - 1) << 16)); + OUT_RING((box->x1 & 0xffff) | (box->y1 << 16)); + OUT_RING(((box->x2 - 1) & 0xffff) | ((box->y2 - 1) << 16)); OUT_RING(DR4); OUT_RING(0); ADVANCE_LP_RING(); } - - return 0; } /* XXX: Emitting the counter should really be moved to part of the IRQ @@ -362,8 +365,9 @@ void i915_emit_breadcrumb(struct drm_device *dev) ADVANCE_LP_RING(); } -static int i915_dispatch_cmdbuffer(struct drm_device * dev, - drm_i915_cmdbuffer_t * cmd) +int +i915_dispatch_cmdbuffer(struct drm_device * dev, + drm_i915_cmdbuffer_t *cmd, struct drm_clip_rect *cliprects) { struct drm_i915_private *dev_priv = dev->dev_private; int nbox = cmd->num_cliprects; @@ -379,12 +383,8 @@ static int i915_dispatch_cmdbuffer(struct drm_device * dev, count = nbox ? nbox : 1; for (i = 0; i < count; i++) { - if (i < nbox) { - ret = i915_emit_box(dev, cmd->cliprects, i, - cmd->DR1, cmd->DR4); - if (ret) - return ret; - } + if (i < nbox) + i915_emit_box(dev, &cliprects[i], cmd->DR1, cmd->DR4); ret = i915_emit_cmds(dev, (int __user *)cmd->buf, cmd->sz / 4); if (ret) @@ -395,13 +395,12 @@ static int i915_dispatch_cmdbuffer(struct drm_device * dev, return 0; } -int i915_dispatch_batchbuffer(struct drm_device * dev, - drm_i915_batchbuffer_t * batch) +int +i915_dispatch_batchbuffer(struct drm_device *dev, + drm_i915_batchbuffer_t *batch, struct drm_clip_rect *cliprects) { - drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_clip_rect __user *boxes = batch->cliprects; - int nbox = batch->num_cliprects; - int i = 0, count; + drm_i915_private_t *dev_priv = dev->dev_private; + int nbox = batch->num_cliprects, i = 0, count; if ((batch->start | batch->used) & 0x7) { DRM_ERROR("alignment\n"); @@ -410,15 +409,13 @@ int i915_dispatch_batchbuffer(struct drm_device * dev, inteldrm_update_ring(dev_priv); + /* XXX use gem code */ count = nbox ? nbox : 1; for (i = 0; i < count; i++) { - if (i < nbox) { - int ret = i915_emit_box(dev, boxes, i, - batch->DR1, batch->DR4); - if (ret) - return ret; - } + if (i < nbox) + i915_emit_box(dev, &cliprects[i], + batch->DR1, batch->DR4); if (!IS_I830(dev_priv) && !IS_845G(dev_priv)) { BEGIN_LP_RING(2); @@ -451,7 +448,7 @@ int i915_flush_ioctl(struct drm_device *dev, void *data, drm_i915_private_t *dev_priv = dev->dev_private; int ret; - LOCK_TEST_WITH_RETURN(dev, file_priv); + RING_LOCK_TEST_WITH_RETURN(dev, file_priv); DRM_LOCK(); inteldrm_update_ring(dev_priv); @@ -464,9 +461,11 @@ int i915_flush_ioctl(struct drm_device *dev, void *data, int i915_batchbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; - drm_i915_batchbuffer_t *batch = data; - int ret; + struct drm_i915_private *dev_priv = + (drm_i915_private_t *)dev->dev_private; + drm_i915_batchbuffer_t *batch = data; + struct drm_clip_rect *cliprects = NULL; + int i, ret; if (!dev_priv->allow_batchbuffer) { DRM_ERROR("Batchbuffer ioctl disabled\n"); @@ -476,46 +475,94 @@ int i915_batchbuffer(struct drm_device *dev, void *data, DRM_DEBUG("i915 batchbuffer, start %x used %d cliprects %d\n", batch->start, batch->used, batch->num_cliprects); + RING_LOCK_TEST_WITH_RETURN(dev, file_priv); + if (batch->num_cliprects < 0) - return EINVAL; + return (EINVAL); - LOCK_TEST_WITH_RETURN(dev, file_priv); + if (batch->num_cliprects) { + if (SIZE_MAX / batch->num_cliprects < sizeof(*cliprects)) + return (EINVAL); + cliprects = drm_alloc(batch->num_cliprects * + sizeof(*cliprects)); + if (cliprects == NULL) + return (ENOMEM); + + ret = copyin((void *)(uintptr_t)batch->cliprects, cliprects, + sizeof(*cliprects) * batch->num_cliprects); + if (ret != 0) + goto free_cliprects; + + for (i = 0; i < batch->num_cliprects; i++) { + if (cliprects[i].y2 <= cliprects[i].y1 || + cliprects[i].x2 <= cliprects[i].x1 || + cliprects[i].y2 <= 0 || cliprects[i].x2 <= 0) { + ret = EINVAL; + goto free_cliprects; + } + } + } DRM_LOCK(); - ret = i915_dispatch_batchbuffer(dev, batch); + ret = i915_dispatch_batchbuffer(dev, batch, cliprects); DRM_UNLOCK(); if (dev_priv->sarea_priv != NULL) dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); - return ret; +free_cliprects: + drm_free(cliprects); + return (ret); } int i915_cmdbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *)dev->dev_private; - drm_i915_cmdbuffer_t *cmdbuf = data; - int ret; + struct drm_i915_private *dev_priv = + (drm_i915_private_t *)dev->dev_private; + drm_i915_cmdbuffer_t *cmdbuf = data; + struct drm_clip_rect *cliprects = NULL; + int i, ret; DRM_DEBUG("i915 cmdbuffer, buf %p sz %d cliprects %d\n", cmdbuf->buf, cmdbuf->sz, cmdbuf->num_cliprects); + RING_LOCK_TEST_WITH_RETURN(dev, file_priv); + if (cmdbuf->num_cliprects < 0) return EINVAL; - LOCK_TEST_WITH_RETURN(dev, file_priv); + if (cmdbuf->num_cliprects) { + if (SIZE_MAX / cmdbuf->num_cliprects < sizeof(*cliprects)) + return (EINVAL); + cliprects = drm_alloc(cmdbuf->num_cliprects * + sizeof(*cliprects)); + if (cliprects == NULL) + return (ENOMEM); + + ret = copyin((void *)(uintptr_t)cmdbuf->cliprects, cliprects, + sizeof(*cliprects) * cmdbuf->num_cliprects); + if (ret != 0) + goto free_cliprects; + + for (i = 0; i < cmdbuf->num_cliprects; i++) { + if (cliprects[i].y2 <= cliprects[i].y1 || + cliprects[i].x2 <= cliprects[i].x1 || + cliprects[i].y2 <= 0 || cliprects[i].x2 <= 0) { + ret = EINVAL; + goto free_cliprects; + } + } + } DRM_LOCK(); - ret = i915_dispatch_cmdbuffer(dev, cmdbuf); + ret = i915_dispatch_cmdbuffer(dev, cmdbuf, cliprects); DRM_UNLOCK(); - if (ret) { - DRM_ERROR("i915_dispatch_cmdbuffer failed\n"); - return ret; - } - - if (dev_priv->sarea_priv != NULL) + if (ret == 0 && dev_priv->sarea_priv != NULL) dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); - return 0; + +free_cliprects: + drm_free(cliprects); + return (ret); } int i915_getparam(struct drm_device *dev, void *data, @@ -544,13 +591,20 @@ int i915_getparam(struct drm_device *dev, void *data, value = dev_priv->pci_device; break; case I915_PARAM_HAS_GEM: +#ifdef INTELDRM_GEM + value = 1; +#else value = 0; +#endif /* INTELDRM_GEM */ break; case I915_PARAM_NUM_FENCES_AVAIL: - value = 0; + value = dev_priv->num_fence_regs - dev_priv->fence_reg_start; + break; + case I915_PARAM_HAS_EXECBUF2: + value = 1; break; default: - DRM_ERROR("Unknown parameter %d\n", param->param); + DRM_DEBUG("Unknown parameter %d\n", param->param); return EINVAL; } @@ -582,8 +636,15 @@ int i915_setparam(struct drm_device *dev, void *data, case I915_SETPARAM_ALLOW_BATCHBUFFER: dev_priv->allow_batchbuffer = param->value; break; + case I915_SETPARAM_NUM_USED_FENCES: + if (param->value > dev_priv->num_fence_regs || + param->value < 0) + return EINVAL; + /* Userspace can use first N regs */ + dev_priv->fence_reg_start = param->value; + break; default: - DRM_ERROR("unknown parameter %d\n", param->param); + DRM_DEBUG("unknown parameter %d\n", param->param); return EINVAL; } diff --git a/sys/dev/pci/drm/i915_drv.c b/sys/dev/pci/drm/i915_drv.c index 651147033ed..8ebab115ff6 100644 --- a/sys/dev/pci/drm/i915_drv.c +++ b/sys/dev/pci/drm/i915_drv.c @@ -1,9 +1,22 @@ -/* i915_drv.c -- Intel i915 driver -*- linux-c -*- - * Created: Wed Feb 14 17:10:04 2001 by gareth@valinux.com +/* + * Copyright (c) 2008-2009 Owain G. Ainsworth <oga@openbsd.org> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ /*- + * Copyright © 2008 Intel Corporation * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. - * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California. + * copyright 2000 VA Linux Systems, Inc., Sunnyvale, California. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a @@ -27,6 +40,7 @@ * * Authors: * Gareth Hughes <gareth@valinux.com> + * Eric Anholt <eric@anholt.net> * */ @@ -35,14 +49,36 @@ #include "i915_drm.h" #include "i915_drv.h" +#include <machine/pmap.h> + +#include <sys/queue.h> +#include <sys/workq.h> +#if 0 +# define INTELDRM_WATCH_COHERENCY +# define WATCH_INACTIVE +#endif + +#define I915_GEM_GPU_DOMAINS (~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT)) + int inteldrm_probe(struct device *, void *, void *); void inteldrm_attach(struct device *, struct device *, void *); int inteldrm_detach(struct device *, int); -int inteldrm_ioctl(struct drm_device *, u_long, caddr_t, struct drm_file *); int inteldrm_activate(struct device *, int); +int inteldrm_ioctl(struct drm_device *, u_long, caddr_t, struct drm_file *); +int inteldrm_intr(void *); void inteldrm_lastclose(struct drm_device *); void inteldrm_wrap_ring(struct drm_i915_private *); +int inteldrm_gmch_match(struct pci_attach_args *); +void inteldrm_chipset_flush(struct drm_i915_private *); +void inteldrm_timeout(void *); +void inteldrm_hangcheck(void *); +void inteldrm_hung(void *, void *); +void inteldrm_965_reset(struct drm_i915_private *, u_int8_t); +int inteldrm_fault(struct drm_obj *, struct uvm_faultinfo *, off_t, + vaddr_t, vm_page_t *, int, int, vm_prot_t, int ); +void inteldrm_wipe_mappings(struct drm_obj *); +void inteldrm_purge_obj(struct drm_obj *); /* For reset and suspend */ int inteldrm_save_state(struct drm_i915_private *); @@ -64,6 +100,112 @@ u_int8_t i915_read_ar(struct drm_i915_private *, u_int16_t, void i915_save_palette(struct drm_i915_private *, enum pipe); void i915_restore_palette(struct drm_i915_private *, enum pipe); +void i915_alloc_ifp(struct drm_i915_private *, struct pci_attach_args *); +void i965_alloc_ifp(struct drm_i915_private *, struct pci_attach_args *); + +void inteldrm_detect_bit_6_swizzle(drm_i915_private_t *, + struct pci_attach_args *); + +int inteldrm_setup_mchbar(struct drm_i915_private *, + struct pci_attach_args *); +void inteldrm_teardown_mchbar(struct drm_i915_private *, + struct pci_attach_args *, int); + +/* Ioctls */ +int i915_gem_init_ioctl(struct drm_device *, void *, struct drm_file *); +int i915_gem_create_ioctl(struct drm_device *, void *, struct drm_file *); +int i915_gem_pread_ioctl(struct drm_device *, void *, struct drm_file *); +int i915_gem_pwrite_ioctl(struct drm_device *, void *, struct drm_file *); +int i915_gem_set_domain_ioctl(struct drm_device *, void *, + struct drm_file *); +int i915_gem_execbuffer2(struct drm_device *, void *, struct drm_file *); +int i915_gem_pin_ioctl(struct drm_device *, void *, struct drm_file *); +int i915_gem_unpin_ioctl(struct drm_device *, void *, struct drm_file *); +int i915_gem_busy_ioctl(struct drm_device *, void *, struct drm_file *); +int i915_gem_entervt_ioctl(struct drm_device *, void *, struct drm_file *); +int i915_gem_leavevt_ioctl(struct drm_device *, void *, struct drm_file *); +int i915_gem_get_aperture_ioctl(struct drm_device *, void *, + struct drm_file *); +int i915_gem_set_tiling(struct drm_device *, void *, struct drm_file *); +int i915_gem_get_tiling(struct drm_device *, void *, struct drm_file *); +int i915_gem_gtt_map_ioctl(struct drm_device *, void *, struct drm_file *); +int i915_gem_madvise_ioctl(struct drm_device *, void *, struct drm_file *); + +/* GEM memory manager functions */ +int i915_gem_init_object(struct drm_obj *); +void i915_gem_free_object(struct drm_obj *); +int i915_gem_object_pin(struct drm_obj *, uint32_t, int); +void i915_gem_object_unpin(struct drm_obj *); +void i915_gem_retire_requests(struct drm_i915_private *); +void i915_gem_retire_request(struct drm_i915_private *, + struct inteldrm_request *); +void i915_gem_retire_work_handler(void *, void*); +int i915_gem_idle(struct drm_i915_private *); +void i915_gem_object_move_to_active(struct drm_obj *); +void i915_gem_object_move_off_active(struct drm_obj *); +void i915_gem_object_move_to_inactive(struct drm_obj *); +uint32_t i915_add_request(struct drm_i915_private *); +void inteldrm_process_flushing(struct drm_i915_private *, u_int32_t); +void i915_move_to_tail(struct inteldrm_obj *, struct i915_gem_list *); +void i915_list_remove(struct inteldrm_obj *); +int i915_gem_init_hws(struct drm_i915_private *); +void i915_gem_cleanup_hws(struct drm_i915_private *); +int i915_gem_init_ringbuffer(struct drm_i915_private *); +int inteldrm_start_ring(struct drm_i915_private *); +void i915_gem_cleanup_ringbuffer(struct drm_i915_private *); +int i915_gem_ring_throttle(struct drm_device *, struct drm_file *); +int i915_gem_evict_inactive(struct drm_i915_private *); +int i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *, + u_int32_t, struct drm_i915_gem_relocation_entry **); +int i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *, + u_int32_t, struct drm_i915_gem_relocation_entry *); +void i915_dispatch_gem_execbuffer(struct drm_device *, + struct drm_i915_gem_execbuffer2 *, uint64_t); +void i915_gem_object_set_to_gpu_domain(struct drm_obj *); +int i915_gem_object_pin_and_relocate(struct drm_obj *, + struct drm_file *, struct drm_i915_gem_exec_object2 *, + struct drm_i915_gem_relocation_entry *); +int i915_gem_object_bind_to_gtt(struct drm_obj *, bus_size_t, int); +int i915_wait_request(struct drm_i915_private *, uint32_t, int); +u_int32_t i915_gem_flush(struct drm_i915_private *, uint32_t, uint32_t); +int i915_gem_object_unbind(struct drm_obj *, int); + +struct drm_obj *i915_gem_find_inactive_object(struct drm_i915_private *, + size_t); + +int i915_gem_evict_everything(struct drm_i915_private *, int); +int i915_gem_evict_something(struct drm_i915_private *, size_t, int); +int i915_gem_object_set_to_gtt_domain(struct drm_obj *, int, int); +int i915_gem_object_set_to_cpu_domain(struct drm_obj *, int, int); +int i915_gem_object_flush_gpu_write_domain(struct drm_obj *, int, int); +int i915_gem_get_fence_reg(struct drm_obj *, int); +int i915_gem_object_put_fence_reg(struct drm_obj *, int); +bus_size_t i915_gem_get_gtt_alignment(struct drm_obj *); + +bus_size_t i915_get_fence_size(struct drm_i915_private *, bus_size_t); +int i915_tiling_ok(struct drm_device *, int, int, int); +int i915_gem_object_fence_offset_ok(struct drm_obj *, int); +void i965_write_fence_reg(struct inteldrm_fence *); +void i915_write_fence_reg(struct inteldrm_fence *); +void i830_write_fence_reg(struct inteldrm_fence *); +void i915_gem_bit_17_swizzle(struct drm_obj *); +void i915_gem_save_bit_17_swizzle(struct drm_obj *); +int inteldrm_swizzle_page(struct vm_page *page); + +/* Debug functions, mostly called from ddb */ +void i915_gem_seqno_info(int); +void i915_interrupt_info(int); +void i915_gem_fence_regs_info(int); +void i915_hws_info(int); +void i915_batchbuffer_info(int); +void i915_ringbuffer_data(int); +void i915_ringbuffer_info(int); +#ifdef WATCH_INACTIVE +void inteldrm_verify_inactive(struct drm_i915_private *, char *, int); +#else +#define inteldrm_verify_inactive(dev,file,line) +#endif + const static struct drm_pcidev inteldrm_pciidlist[] = { {PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82830M_IGD, CHIP_I830|CHIP_M}, @@ -104,7 +246,7 @@ const static struct drm_pcidev inteldrm_pciidlist[] = { {PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82Q33_IGD_1, CHIP_G33|CHIP_I9XX|CHIP_HWS}, {PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82GM45_IGD_1, - CHIP_GM45|CHIP_I965|CHIP_I9XX|CHIP_M|CHIP_HWS}, + CHIP_G4X|CHIP_GM45|CHIP_I965|CHIP_I9XX|CHIP_M|CHIP_HWS}, {PCI_VENDOR_INTEL, 0x2E02, CHIP_G4X|CHIP_I965|CHIP_I9XX|CHIP_HWS}, {PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82Q45_IGD_1, @@ -118,6 +260,7 @@ const static struct drm_pcidev inteldrm_pciidlist[] = { static const struct drm_driver_info inteldrm_driver = { .buf_priv_size = 1, /* No dev_priv */ + .file_priv_size = sizeof(struct inteldrm_file), .ioctl = inteldrm_ioctl, .lastclose = inteldrm_lastclose, .vblank_pipes = 2, @@ -127,6 +270,13 @@ static const struct drm_driver_info inteldrm_driver = { .irq_install = i915_driver_irq_install, .irq_uninstall = i915_driver_irq_uninstall, +#ifdef INTELDRM_GEM + .gem_init_object = i915_gem_init_object, + .gem_free_object = i915_gem_free_object, + .gem_fault = inteldrm_fault, + .gem_size = sizeof(struct inteldrm_obj), +#endif /* INTELDRM_GEM */ + .name = DRIVER_NAME, .desc = DRIVER_DESC, .date = DRIVER_DATE, @@ -135,7 +285,10 @@ static const struct drm_driver_info inteldrm_driver = { .patchlevel = DRIVER_PATCHLEVEL, .flags = DRIVER_AGP | DRIVER_AGP_REQUIRE | - DRIVER_MTRR | DRIVER_IRQ, + DRIVER_MTRR | DRIVER_IRQ +#ifdef INTELDRM_GEM + | DRIVER_GEM, +#endif /* INTELDRM_GEM */ }; int @@ -145,13 +298,30 @@ inteldrm_probe(struct device *parent, void *match, void *aux) inteldrm_pciidlist)); } +/* + * We're intel IGD, bus 0 function 0 dev 0 should be the GMCH, so it should + * be Intel + */ +int +inteldrm_gmch_match(struct pci_attach_args *pa) +{ + if (pa->pa_bus == 0 && pa->pa_device == 0 && pa->pa_function == 0 && + PCI_VENDOR(pa->pa_id) == PCI_VENDOR_INTEL && + PCI_CLASS(pa->pa_class) == PCI_CLASS_BRIDGE && + PCI_SUBCLASS(pa->pa_class) == PCI_SUBCLASS_BRIDGE_HOST) + return (1); + return (0); +} + void inteldrm_attach(struct device *parent, struct device *self, void *aux) { struct drm_i915_private *dev_priv = (struct drm_i915_private *)self; - struct pci_attach_args *pa = aux; + struct pci_attach_args *pa = aux, bpa; struct vga_pci_bar *bar; + struct drm_device *dev; const struct drm_pcidev *id_entry; + int i; id_entry = drm_find_description(PCI_VENDOR(pa->pa_id), PCI_PRODUCT(pa->pa_id), inteldrm_pciidlist); @@ -163,7 +333,7 @@ inteldrm_attach(struct device *parent, struct device *self, void *aux) dev_priv->dmat = pa->pa_dmat; dev_priv->bst = pa->pa_memt; - /* Add register map (needed for suspend/resume) */ + /* we need to use this api for now due to sharing with intagp */ bar = vga_pci_bar_info((struct vga_pci_softc *)parent, (IS_I9XX(dev_priv) ? 0 : 1)); if (bar == NULL) { @@ -183,10 +353,96 @@ inteldrm_attach(struct device *parent, struct device *self, void *aux) return; } + /* + * set up interrupt handler, note that we don't switch the interrupt + * on until the X server talks to us, kms will change this. + */ + dev_priv->irqh = pci_intr_establish(dev_priv->pc, dev_priv->ih, IPL_BIO, + inteldrm_intr, dev_priv, dev_priv->dev.dv_xname); + if (dev_priv->irqh == NULL) { + printf(": couldn't establish interrupt\n"); + return; + } + + /* Unmask the interrupts that we always want on. */ + dev_priv->irq_mask_reg = ~I915_INTERRUPT_ENABLE_FIX; + +#ifdef INTELDRM_GEM + dev_priv->workq = workq_create("intelrel", 1, IPL_BIO); + if (dev_priv->workq == NULL) { + printf("couldn't create workq\n"); + return; + } + + /* GEM init */ + TAILQ_INIT(&dev_priv->mm.active_list); + TAILQ_INIT(&dev_priv->mm.flushing_list); + TAILQ_INIT(&dev_priv->mm.inactive_list); + TAILQ_INIT(&dev_priv->mm.gpu_write_list); + TAILQ_INIT(&dev_priv->mm.request_list); + TAILQ_INIT(&dev_priv->mm.fence_list); + timeout_set(&dev_priv->mm.retire_timer, inteldrm_timeout, dev_priv); + timeout_set(&dev_priv->mm.hang_timer, inteldrm_hangcheck, dev_priv); + dev_priv->mm.next_gem_seqno = 1; + dev_priv->mm.suspended = 1; +#endif /* INTELDRM_GEM */ + + /* For the X server, in kms mode this will not be needed */ + dev_priv->fence_reg_start = 3; + + if (IS_I965G(dev_priv) || IS_I945G(dev_priv) || IS_I945GM(dev_priv) || + IS_G33(dev_priv)) + dev_priv->num_fence_regs = 16; + else + dev_priv->num_fence_regs = 8; + /* Initialise fences to zero, else on some macs we'll get corruption */ + if (IS_I965G(dev_priv)) { + for (i = 0; i < 16; i++) + I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0); + } else { + for (i = 0; i < 8; i++) + I915_WRITE(FENCE_REG_830_0 + (i * 4), 0); + if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) || + IS_G33(dev_priv)) + for (i = 0; i < 8; i++) + I915_WRITE(FENCE_REG_945_8 + (i * 4), 0); + } + + if (pci_find_device(&bpa, inteldrm_gmch_match) == 0) { + printf(": can't find GMCH\n"); + return; + } + + /* Set up the IFP for chipset flushing */ + if (dev_priv->flags & (CHIP_I915G|CHIP_I915GM|CHIP_I945G|CHIP_I945GM)) { + i915_alloc_ifp(dev_priv, &bpa); + } else if (IS_I965G(dev_priv) || IS_G33(dev_priv)) { + i965_alloc_ifp(dev_priv, &bpa); + } else { + int nsegs; + /* + * I8XX has no flush page mechanism, we fake it by writing until + * the cache is empty. allocate a page to scribble on + */ + dev_priv->ifp.i8xx.kva = NULL; + if (bus_dmamem_alloc(pa->pa_dmat, PAGE_SIZE, 0, 0, + &dev_priv->ifp.i8xx.seg, 1, &nsegs, BUS_DMA_WAITOK) == 0) { + if (bus_dmamem_map(pa->pa_dmat, &dev_priv->ifp.i8xx.seg, + 1, PAGE_SIZE, &dev_priv->ifp.i8xx.kva, 0) != 0) { + bus_dmamem_free(pa->pa_dmat, + &dev_priv->ifp.i8xx.seg, nsegs); + dev_priv->ifp.i8xx.kva = NULL; + } + } + } + +#ifdef INTELDRM_GEM + inteldrm_detect_bit_6_swizzle(dev_priv, &bpa); +#endif /* INTELDRM_GEM */ /* Init HWS */ if (!I915_NEED_GFX_HWS(dev_priv)) { if (i915_init_phys_hws(dev_priv, pa->pa_dmat) != 0) { - printf(": couldn't initialize hardware status page\n"); + printf(": couldn't alloc HWS page\n"); return; } } @@ -197,6 +453,24 @@ inteldrm_attach(struct device *parent, struct device *self, void *aux) /* All intel chipsets need to be treated as agp, so just pass one */ dev_priv->drmdev = drm_attach_pci(&inteldrm_driver, pa, 1, self); + + dev = (struct drm_device *)dev_priv->drmdev; + +#ifdef INTELDRM_GEM + /* XXX would be a lot nicer to get agp info before now */ + uvm_page_physload_flags(atop(dev->agp->base), atop(dev->agp->base + + dev->agp->info.ai_aperture_size), atop(dev->agp->base), + atop(dev->agp->base + dev->agp->info.ai_aperture_size), 0, + PHYSLOAD_DEVICE); + /* array of vm pages that physload introduced. */ + dev_priv->pgs = PHYS_TO_VM_PAGE(dev->agp->base); + KASSERT(dev_priv->pgs != NULL); + /* XXX wc and do earlier */ + if (bus_space_map(dev_priv->bst, dev->agp->base, + dev->agp->info.ai_aperture_size, BUS_SPACE_MAP_LINEAR, + &dev_priv->aperture_bsh) != 0) + panic("can't map aperture"); +#endif /* INTELDRM_GEM */ } int @@ -204,6 +478,7 @@ inteldrm_detach(struct device *self, int flags) { struct drm_i915_private *dev_priv = (struct drm_i915_private *)self; + /* this will quiesce any dma that's going on and kill the timeouts. */ if (dev_priv->drmdev != NULL) { config_detach(dev_priv->drmdev, flags); dev_priv->drmdev = NULL; @@ -211,6 +486,18 @@ inteldrm_detach(struct device *self, int flags) i915_free_hws(dev_priv, dev_priv->dmat); + if (IS_I9XX(dev_priv) && dev_priv->ifp.i9xx.bsh != NULL) { + bus_space_unmap(dev_priv->ifp.i9xx.bst, dev_priv->ifp.i9xx.bsh, + PAGE_SIZE); + } else if (dev_priv->flags & (CHIP_I830 | CHIP_I845G | CHIP_I85X | + CHIP_I865G) && dev_priv->ifp.i8xx.kva != NULL) { + bus_dmamem_unmap(dev_priv->dmat, dev_priv->ifp.i8xx.kva, + PAGE_SIZE); + bus_dmamem_free(dev_priv->dmat, &dev_priv->ifp.i8xx.seg, 1); + } + + pci_intr_disestablish(dev_priv->pc, dev_priv->irqh); + if (dev_priv->regs != NULL) vga_pci_bar_unmap(dev_priv->regs); @@ -263,6 +550,36 @@ inteldrm_ioctl(struct drm_device *dev, u_long cmd, caddr_t data, return (i915_cmdbuffer(dev, data, file_priv)); case DRM_IOCTL_I915_GET_VBLANK_PIPE: return (i915_vblank_pipe_get(dev, data, file_priv)); +#ifdef INTELDRM_GEM + case DRM_IOCTL_I915_GEM_EXECBUFFER2: + return (i915_gem_execbuffer2(dev, data, file_priv)); + case DRM_IOCTL_I915_GEM_BUSY: + return (i915_gem_busy_ioctl(dev, data, file_priv)); + case DRM_IOCTL_I915_GEM_THROTTLE: + return (i915_gem_ring_throttle(dev, file_priv)); + case DRM_IOCTL_I915_GEM_MMAP: + return (i915_gem_gtt_map_ioctl(dev, data, file_priv)); + case DRM_IOCTL_I915_GEM_CREATE: + return (i915_gem_create_ioctl(dev, data, file_priv)); + case DRM_IOCTL_I915_GEM_PREAD: + return (i915_gem_pread_ioctl(dev, data, file_priv)); + case DRM_IOCTL_I915_GEM_PWRITE: + return (i915_gem_pwrite_ioctl(dev, data, file_priv)); + case DRM_IOCTL_I915_GEM_SET_DOMAIN: + return (i915_gem_set_domain_ioctl(dev, data, + file_priv)); + case DRM_IOCTL_I915_GEM_SET_TILING: + return (i915_gem_set_tiling(dev, data, file_priv)); + case DRM_IOCTL_I915_GEM_GET_TILING: + return (i915_gem_get_tiling(dev, data, file_priv)); + case DRM_IOCTL_I915_GEM_GET_APERTURE: + return (i915_gem_get_aperture_ioctl(dev, data, + file_priv)); + case DRM_IOCTL_I915_GEM_MADVISE: + return (i915_gem_madvise_ioctl(dev, data, file_priv)); +#endif /* INTELDRM_GEM */ + default: + break; } } @@ -279,11 +596,84 @@ inteldrm_ioctl(struct drm_device *dev, u_long cmd, caddr_t data, case DRM_IOCTL_I915_DESTROY_HEAP: case DRM_IOCTL_I915_SET_VBLANK_PIPE: return (0); +#ifdef INTELDRM_GEM + case DRM_IOCTL_I915_GEM_INIT: + return (i915_gem_init_ioctl(dev, data, file_priv)); + case DRM_IOCTL_I915_GEM_ENTERVT: + return (i915_gem_entervt_ioctl(dev, data, file_priv)); + case DRM_IOCTL_I915_GEM_LEAVEVT: + return (i915_gem_leavevt_ioctl(dev, data, file_priv)); + case DRM_IOCTL_I915_GEM_PIN: + return (i915_gem_pin_ioctl(dev, data, file_priv)); + case DRM_IOCTL_I915_GEM_UNPIN: + return (i915_gem_unpin_ioctl(dev, data, file_priv)); +#endif /* INTELDRM_GEM */ } } return (EINVAL); } +int +inteldrm_intr(void *arg) +{ + drm_i915_private_t *dev_priv = arg; + struct drm_device *dev = (struct drm_device *)dev_priv->drmdev; + u_int32_t iir, pipea_stats = 0, pipeb_stats = 0; + + /* we're not set up, don't poke the hw */ + if (dev_priv->hw_status_page == NULL) + return (0); + /* + * lock is to protect from writes to PIPESTAT and IMR from other cores. + */ + mtx_enter(&dev_priv->user_irq_lock); + iir = I915_READ(IIR); + if (iir == 0) { + mtx_leave(&dev_priv->user_irq_lock); + return (0); + } + + /* + * Clear the PIPE(A|B)STAT regs before the IIR + */ + if (iir & I915_DISPLAY_PIPE_A_EVENT_INTERRUPT) { + pipea_stats = I915_READ(PIPEASTAT); + I915_WRITE(PIPEASTAT, pipea_stats); + } + if (iir & I915_DISPLAY_PIPE_B_EVENT_INTERRUPT) { + pipeb_stats = I915_READ(PIPEBSTAT); + I915_WRITE(PIPEBSTAT, pipeb_stats); + } +#ifdef INTELDRM_GEM + if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT) + inteldrm_error(dev_priv); +#endif /* INTELDRM_GEM */ + + I915_WRITE(IIR, iir); + (void)I915_READ(IIR); /* Flush posted writes */ + + if (dev_priv->sarea_priv != NULL) + dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); + + if (iir & I915_USER_INTERRUPT) { + wakeup(dev_priv); +#ifdef INTELDRM_GEM + dev_priv->mm.hang_cnt = 0; + timeout_add_msec(&dev_priv->mm.hang_timer, 750); +#endif /* INTELDRM_GEM */ + } + + mtx_leave(&dev_priv->user_irq_lock); + + if (pipea_stats & I915_VBLANK_INTERRUPT_STATUS) + drm_handle_vblank(dev, 0); + + if (pipeb_stats & I915_VBLANK_INTERRUPT_STATUS) + drm_handle_vblank(dev, 1); + + return (1); +} + u_int32_t inteldrm_read_hws(struct drm_i915_private *dev_priv, int reg) { @@ -339,8 +729,7 @@ inteldrm_wait_ring(struct drm_i915_private *dev_priv, int n) last_head = ring->head; last_acthd = acthd; - tsleep(dev_priv, PZERO | PCATCH, "i915wt", - hz / 100); + delay(10); } return (EBUSY); @@ -416,15 +805,3780 @@ inteldrm_update_ring(struct drm_i915_private *dev_priv) } void +i915_alloc_ifp(struct drm_i915_private *dev_priv, struct pci_attach_args *bpa) +{ + bus_addr_t addr; + u_int32_t reg; + + dev_priv->ifp.i9xx.bst = bpa->pa_memt; + + reg = pci_conf_read(bpa->pa_pc, bpa->pa_tag, I915_IFPADDR); + if (reg & 0x1) { + addr = (bus_addr_t)reg; + addr &= ~0x1; + /* XXX extents ... need data on whether bioses alloc or not. */ + if (bus_space_map(bpa->pa_memt, addr, PAGE_SIZE, 0, + &dev_priv->ifp.i9xx.bsh) != 0) + goto nope; + return; + } else if (bpa->pa_memex == NULL || extent_alloc(bpa->pa_memex, + PAGE_SIZE, PAGE_SIZE, 0, 0, 0, &addr) || bus_space_map(bpa->pa_memt, + addr, PAGE_SIZE, 0, &dev_priv->ifp.i9xx.bsh)) + goto nope; + + pci_conf_write(bpa->pa_pc, bpa->pa_tag, I915_IFPADDR, addr | 0x1); + + return; + +nope: + dev_priv->ifp.i9xx.bsh = NULL; + printf(": no ifp "); +} + +void +i965_alloc_ifp(struct drm_i915_private *dev_priv, struct pci_attach_args *bpa) +{ + bus_addr_t addr; + u_int32_t lo, hi; + + dev_priv->ifp.i9xx.bst = bpa->pa_memt; + + hi = pci_conf_read(bpa->pa_pc, bpa->pa_tag, I965_IFPADDR + 4); + lo = pci_conf_read(bpa->pa_pc, bpa->pa_tag, I965_IFPADDR); + if (lo & 0x1) { + addr = (((u_int64_t)hi << 32) | lo); + addr &= ~0x1; + /* XXX extents ... need data on whether bioses alloc or not. */ + if (bus_space_map(bpa->pa_memt, addr, PAGE_SIZE, 0, + &dev_priv->ifp.i9xx.bsh) != 0) + goto nope; + return; + } else if (bpa->pa_memex == NULL || extent_alloc(bpa->pa_memex, + PAGE_SIZE, PAGE_SIZE, 0, 0, 0, &addr) || bus_space_map(bpa->pa_memt, + addr, PAGE_SIZE, 0, &dev_priv->ifp.i9xx.bsh)) + goto nope; + + pci_conf_write(bpa->pa_pc, bpa->pa_tag, I965_IFPADDR + 4, + upper_32_bits(addr)); + pci_conf_write(bpa->pa_pc, bpa->pa_tag, I965_IFPADDR, + (addr & 0xffffffff) | 0x1); + + return; + +nope: + dev_priv->ifp.i9xx.bsh = NULL; + printf(": no ifp "); +} + +void +inteldrm_chipset_flush(struct drm_i915_private *dev_priv) +{ + /* + * Write to this flush page flushes the chipset write cache. + * The write will return when it is done. + */ + if (IS_I9XX(dev_priv)) { + if (dev_priv->ifp.i9xx.bsh != NULL) + bus_space_write_4(dev_priv->ifp.i9xx.bst, + dev_priv->ifp.i9xx.bsh, 0, 1); + } else { + /* + * I8XX don't have a flush page mechanism, but do have the + * cache. Do it the bruteforce way. we write 1024 byes into + * the cache, then clflush them out so they'll kick the stuff + * we care about out of the chipset cache. + */ + if (dev_priv->ifp.i8xx.kva != NULL) { + memset(dev_priv->ifp.i8xx.kva, 0, 1024); + agp_flush_cache_range((vaddr_t)dev_priv->ifp.i8xx.kva, + 1024); + } + } +} + +void inteldrm_lastclose(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + drm_i915_private_t *dev_priv = dev->dev_private; +#ifdef INTELDRM_GEM + struct vm_page *p; + int ret; + + ret = i915_gem_idle(dev_priv); + if (ret) + DRM_ERROR("failed to idle hardware: %d\n", ret); + + if (dev_priv->agpdmat != NULL) { + /* + * make sure we nuke everything, we may have mappings that we've + * unrefed, but uvm has a reference to them for maps. Make sure + * they get unbound and any accesses will segfault. + * XXX only do ones in GEM. + */ + for (p = dev_priv->pgs; p < dev_priv->pgs + + (dev->agp->info.ai_aperture_size / PAGE_SIZE); p++) + pmap_page_protect(p, VM_PROT_NONE); + agp_bus_dma_destroy((struct agp_softc *)dev->agp->agpdev, + dev_priv->agpdmat); + } +#endif /* INTELDRM_GEM */ + dev_priv->agpdmat = NULL; + dev_priv->sarea_priv = NULL; i915_dma_cleanup(dev); } +#ifdef INTELDRM_GEM + +int +i915_gem_init_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_gem_init *args = data; + + DRM_LOCK(); + + if (args->gtt_start >= args->gtt_end || + args->gtt_end > dev->agp->info.ai_aperture_size || + (args->gtt_start & PAGE_MASK) != 0 || + (args->gtt_end & PAGE_MASK) != 0) { + DRM_UNLOCK(); + return (EINVAL); + } + /* + * putting stuff in the last page of the aperture can cause nasty + * problems with prefetch going into unassigned memory. Since we put + * a scratch page on all unused aperture pages, just leave the last + * page as a spill to prevent gpu hangs. + */ + if (args->gtt_end == dev->agp->info.ai_aperture_size) + args->gtt_end -= 4096; + + if (agp_bus_dma_init((struct agp_softc *)dev->agp->agpdev, + dev->agp->base + args->gtt_start, dev->agp->base + args->gtt_end, + &dev_priv->agpdmat) != 0) { + DRM_UNLOCK(); + return (ENOMEM); + } + + dev->gtt_total = (uint32_t)(args->gtt_end - args->gtt_start); + + DRM_UNLOCK(); + + return 0; +} + +int +i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_i915_gem_get_aperture *args = data; + + args->aper_size = dev->gtt_total; + args->aper_available_size = (args->aper_size - + atomic_read(&dev->pin_memory)); + + return (0); +} + +/** + * Creates a new mm object and returns a handle to it. + */ +int +i915_gem_create_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_i915_gem_create *args = data; + struct drm_obj *obj; + int handle, ret; + + args->size = round_page(args->size); + + /* Allocate the new object */ + obj = drm_gem_object_alloc(dev, args->size); + if (obj == NULL) + return (ENOMEM); + + ret = drm_handle_create(file_priv, obj, &handle); + /* handle has a reference now, drop ours. */ + DRM_LOCK(); + drm_gem_object_unreference(obj); + DRM_UNLOCK(); + + if (ret == 0) + args->handle = handle; + + return (ret); +} + +/** + * Reads data from the object referenced by handle. + * + * On error, the contents of *data are undefined. + */ +int +i915_gem_pread_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_gem_pread *args = data; + struct drm_obj *obj; + struct inteldrm_obj *obj_priv; + char *vaddr; + bus_space_handle_t bsh; + bus_size_t bsize; + voff_t offset; + int ret; + + obj = drm_gem_object_lookup(dev, file_priv, args->handle); + if (obj == NULL) + return (EBADF); + obj_priv = (struct inteldrm_obj *)obj; + + /* + * Bounds check source. + */ + DRM_LOCK(); + if (args->offset > obj->size || args->size > obj->size || + args->offset + args->size > obj->size) { + ret = EINVAL; + goto out; + } + + ret = i915_gem_object_pin(obj, 0, 1); + if (ret) + goto out; + ret = i915_gem_object_set_to_gtt_domain(obj, 0, 1); + if (ret) + goto unpin; + + obj_priv = (struct inteldrm_obj *)obj; + offset = obj_priv->gtt_offset + args->offset; + + bsize = round_page(offset + args->size) - trunc_page(offset); + + if ((ret = bus_space_subregion(dev_priv->bst, dev_priv->aperture_bsh, + trunc_page(offset), bsize, &bsh)) != 0) + goto unpin; + vaddr = bus_space_vaddr(dev->bst, bsh); + if (vaddr == NULL) { + ret = EFAULT; + goto unpin; + } + + ret = copyout(vaddr + (offset & PAGE_MASK), + (char *)(uintptr_t)args->data_ptr, args->size); + + if (ret) + goto unpin; + +unpin: + i915_gem_object_unpin(obj); +out: + drm_gem_object_unreference(obj); + DRM_UNLOCK(); + + return (ret); +} + + +/** + * Writes data to the object referenced by handle. + * + * On error, the contents of the buffer that were to be modified are undefined. + */ +int +i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_gem_pwrite *args = data; + struct drm_obj *obj; + struct inteldrm_obj *obj_priv; + char *vaddr; + bus_space_handle_t bsh; + bus_size_t bsize; + off_t offset; + int ret = 0; + + obj = drm_gem_object_lookup(dev, file_priv, args->handle); + if (obj == NULL) + return (EBADF); + obj_priv = (struct inteldrm_obj *)obj; + + DRM_LOCK(); + /* Bounds check destination. */ + if (args->offset > obj->size || args->size > obj->size || + args->offset + args->size > obj->size) { + ret = EINVAL; + goto out; + } + + ret = i915_gem_object_pin(obj, 0, 1); + if (ret) + goto out; + ret = i915_gem_object_set_to_gtt_domain(obj, 1, 1); + if (ret) + goto done; + + obj_priv = (struct inteldrm_obj *)obj; + offset = obj_priv->gtt_offset + args->offset; + bsize = round_page(offset + args->size) - trunc_page(offset); + + if ((ret = bus_space_subregion(dev_priv->bst, dev_priv->aperture_bsh, + trunc_page(offset), bsize, &bsh)) != 0) + goto done; + vaddr = bus_space_vaddr(dev_priv->bst, bsh); + if (vaddr == NULL) { + ret = EFAULT; + goto done; + } + + ret = copyin((char *)(uintptr_t)args->data_ptr, + vaddr + (offset & PAGE_MASK), args->size); + + +done: + i915_gem_object_unpin(obj); +out: + drm_gem_object_unreference(obj); + DRM_UNLOCK(); + + return (ret); +} + +/** + * Called when user space prepares to use an object with the CPU, either through + * the mmap ioctl's mapping or a GTT mapping. + */ +int +i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_i915_gem_set_domain *args = data; + struct drm_obj *obj; + u_int32_t read_domains = args->read_domains; + u_int32_t write_domain = args->write_domain; + int ret; + + /* + * Only handle setting domains to types we allow the cpu to see. + * while linux allows the CPU domain here, we only allow GTT since that + * is all that we let userland near. + * Also sanity check that having something in the write domain implies + * it's in the read domain, and only that read domain. + */ + if ((write_domain | read_domains) & ~I915_GEM_DOMAIN_GTT || + (write_domain != 0 && read_domains != write_domain)) + return (EINVAL); + + obj = drm_gem_object_lookup(dev, file_priv, args->handle); + if (obj == NULL) + return (EBADF); + + DRM_LOCK(); + ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0, 1); + + drm_gem_object_unreference(obj); + DRM_UNLOCK(); + /* + * Silently promote `you're not bound, there was nothing to do' + * to success, since the client was just asking us to make sure + * everything was done. + */ + return ((ret == EINVAL) ? 0 : ret); +} + +int +i915_gem_gtt_map_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_i915_gem_mmap *args = data; + struct drm_obj *obj; + struct inteldrm_obj *obj_priv; + vaddr_t addr; + voff_t offset; + vsize_t end, nsize; + int ret; + + obj = drm_gem_object_lookup(dev, file_priv, args->handle); + if (obj == NULL) + return (EBADF); + + DRM_LOCK(); + obj_priv = (struct inteldrm_obj *)obj; + + /* Check size. Also ensure that the object is not purgeable */ + if (args->size == 0 || args->offset > obj->size || args->size > + obj->size || (args->offset + args->size) > obj->size || + i915_obj_purgeable(obj_priv)) { + ret = EINVAL; + goto done; + } + + /* bind to the gtt to speed faulting */ + if (obj_priv->dmamap == NULL) { + ret = i915_gem_object_bind_to_gtt(obj, 0, 1); + if (ret) + goto done; + i915_gem_object_move_to_inactive(obj); + } + + + end = round_page(args->offset + args->size); + offset = trunc_page(args->offset); + nsize = end - offset; + + /* + * We give our reference from object_lookup to the mmap, so only + * must free it in the case that the map fails. + */ + addr = uvm_map_hint(curproc, VM_PROT_READ | VM_PROT_WRITE); + ret = uvm_map_p(&curproc->p_vmspace->vm_map, &addr, nsize, &obj->uobj, + offset, 0, UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW, + UVM_INH_SHARE, UVM_ADV_RANDOM, 0), curproc); + +done: + if (ret != 0) + drm_gem_object_unreference(obj); + DRM_UNLOCK(); + + if (ret == 0) + args->addr_ptr = (uint64_t) addr + (args->offset & PAGE_MASK); + + return (ret); +} + +/* called locked */ +void +i915_gem_object_move_to_active(struct drm_obj *obj) +{ + struct drm_device *dev = obj->dev; + drm_i915_private_t *dev_priv = dev->dev_private; + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + struct inteldrm_fence *reg; + u_int32_t seqno = dev_priv->mm.next_gem_seqno; + + /* Add a reference if we're newly entering the active list. */ + if (!inteldrm_is_active(obj_priv)) { + drm_gem_object_reference(obj); + atomic_setbits_int(&obj_priv->io_flags, I915_ACTIVE); + } + + if (inteldrm_needs_fence(obj_priv)) { + reg = &dev_priv->fence_regs[obj_priv->fence_reg]; + reg->last_rendering_seqno = seqno; + } + + /* Move from whatever list we were on to the tail of execution. */ + i915_move_to_tail(obj_priv, &dev_priv->mm.active_list); + obj_priv->last_rendering_seqno = seqno; +} + +void +i915_gem_object_move_off_active(struct drm_obj *obj) +{ + struct drm_device *dev = obj->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + struct inteldrm_fence *reg; + + obj_priv->last_rendering_seqno = 0; + if (inteldrm_needs_fence(obj_priv)) { + reg = &dev_priv->fence_regs[obj_priv->fence_reg]; + reg->last_rendering_seqno = 0; + } +} + +/* called locked */ +void +i915_gem_object_move_to_inactive(struct drm_obj *obj) +{ + struct drm_device *dev = obj->dev; + drm_i915_private_t *dev_priv = dev->dev_private; + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + + inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__); + if (obj_priv->pin_count != 0) + i915_list_remove(obj_priv); + else + i915_move_to_tail(obj_priv, &dev_priv->mm.inactive_list); + + i915_gem_object_move_off_active(obj); + atomic_clearbits_int(&obj_priv->io_flags, I915_FENCED_EXEC); + + KASSERT((obj_priv->io_flags & I915_GPU_WRITE) == 0); + if (inteldrm_is_active(obj_priv)) { + atomic_clearbits_int(&obj_priv->io_flags, + I915_ACTIVE); + drm_gem_object_unreference(obj); + } + inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__); +} + +void +inteldrm_purge_obj(struct drm_obj *obj) +{ + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + + /* + * may sleep. We free here instead of deactivate (which + * the madvise() syscall would do) because in this case + * (userland bo cache and GL_APPLE_object_purgeable objects in + * OpenGL) the pages are defined to be freed if they were cleared + * so kill them and free up the memory + */ + simple_lock(&obj->uao->vmobjlock); + obj->uao->pgops->pgo_flush(obj->uao, 0, obj->size, + PGO_ALLPAGES | PGO_FREE); + simple_unlock(&obj->uao->vmobjlock); + + /* + * If flush failed, it may have halfway through, so just + * always mark as purged + */ + atomic_setbits_int(&obj_priv->io_flags, I915_PURGED); +} + +void +inteldrm_process_flushing(struct drm_i915_private *dev_priv, + u_int32_t flush_domains) +{ + struct inteldrm_obj *obj_priv, *next; + + for (obj_priv = TAILQ_FIRST(&dev_priv->mm.gpu_write_list); + obj_priv != TAILQ_END(&dev_priv->mm.gpu_write_list); + obj_priv = next) { + struct drm_obj *obj = &(obj_priv->obj); + + next = TAILQ_NEXT(obj_priv, write_list); + + if ((obj->write_domain & flush_domains) == obj->write_domain) { + + obj->write_domain = 0; + TAILQ_REMOVE(&dev_priv->mm.gpu_write_list, + obj_priv, write_list); + atomic_clearbits_int(&obj_priv->io_flags, + I915_GPU_WRITE); + i915_gem_object_move_to_active(obj); + /* if we still need the fence, update LRU */ + if (inteldrm_needs_fence(obj_priv)) { + KASSERT(obj_priv->fence_reg != + I915_FENCE_REG_NONE); + /* we have a fence, won't sleep, can't fail */ + i915_gem_get_fence_reg(obj, 1); + } + + } + } +} + +/** + * Creates a new sequence number, emitting a write of it to the status page + * plus an interrupt, which will trigger and interrupt if they are currently + * enabled. + * + * Must be called with struct_lock held. + * + * Returned sequence numbers are nonzero on success. + */ +uint32_t +i915_add_request(struct drm_i915_private *dev_priv) +{ + struct inteldrm_request *request; + uint32_t seqno; + int was_empty; + + request = drm_calloc(1, sizeof(*request)); + if (request == NULL) { + printf("%s: failed to allocate request\n", __func__); + return 0; + } + + /* Grab the seqno we're going to make this request be, and bump the + * next (skipping 0 so it can be the reserved no-seqno value). + */ + seqno = dev_priv->mm.next_gem_seqno; + dev_priv->mm.next_gem_seqno++; + if (dev_priv->mm.next_gem_seqno == 0) + dev_priv->mm.next_gem_seqno++; + + BEGIN_LP_RING(4); + OUT_RING(MI_STORE_DWORD_INDEX); + OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); + OUT_RING(seqno); + + OUT_RING(MI_USER_INTERRUPT); + ADVANCE_LP_RING(); + + DRM_DEBUG("%d\n", seqno); + + /* XXX request timing for throttle */ + request->seqno = seqno; + was_empty = TAILQ_EMPTY(&dev_priv->mm.request_list); + TAILQ_INSERT_TAIL(&dev_priv->mm.request_list, request, list); + + if (dev_priv->mm.suspended == 0) { + if (was_empty) + timeout_add_sec(&dev_priv->mm.retire_timer, 1); + /* XXX was_empty? */ + timeout_add_msec(&dev_priv->mm.hang_timer, 750); + } + return seqno; +} + +/** + * Moves buffers associated only with the given active seqno from the active + * to inactive list, potentially freeing them. + * + * called with and sleeps with the drm_lock. + */ +void +i915_gem_retire_request(struct drm_i915_private *dev_priv, + struct inteldrm_request *request) +{ + struct inteldrm_obj *obj_priv; + + /* Move any buffers on the active list that are no longer referenced + * by the ringbuffer to the flushing/inactive lists as appropriate. + */ + while ((obj_priv = TAILQ_FIRST(&dev_priv->mm.active_list)) != NULL) { + struct drm_obj *obj = &obj_priv->obj; + + /* If the seqno being retired doesn't match the oldest in the + * list, then the oldest in the list must still be newer than + * this seqno. + */ + if (obj_priv->last_rendering_seqno != request->seqno) + return; + + /* + * If we're now clean and can be read from, move inactive, + * else put on the flushing list to signify that we're not + * available quite yet. + */ + if (obj->write_domain != 0) { + KASSERT(inteldrm_is_active(obj_priv)); + i915_move_to_tail(obj_priv, + &dev_priv->mm.flushing_list); + i915_gem_object_move_off_active(obj); + } else { + i915_gem_object_move_to_inactive(obj); + } + } +} + +/** + * This function clears the request list as sequence numbers are passed. + */ +void +i915_gem_retire_requests(struct drm_i915_private *dev_priv) +{ + struct inteldrm_request *request; + uint32_t seqno; + + if (dev_priv->hw_status_page == NULL) + return; + + seqno = i915_get_gem_seqno(dev_priv); + + while ((request = TAILQ_FIRST(&dev_priv->mm.request_list)) != NULL) { + if (i915_seqno_passed(seqno, request->seqno) || + dev_priv->mm.wedged) { + i915_gem_retire_request(dev_priv, request); + + TAILQ_REMOVE(&dev_priv->mm.request_list, request, list); + drm_free(request); + } else + break; + } +} + +void +i915_gem_retire_work_handler(void *arg1, void *unused) +{ + drm_i915_private_t *dev_priv = arg1; + struct drm_device *dev = (struct drm_device *)dev_priv->drmdev; + + DRM_LOCK(); + i915_gem_retire_requests(dev_priv); + if (!TAILQ_EMPTY(&dev_priv->mm.request_list)) + timeout_add_sec(&dev_priv->mm.retire_timer, 1); + DRM_UNLOCK(); +} + +/** + * Waits for a sequence number to be signaled, and cleans up the + * request and object lists appropriately for that event. + * + * Called locked, sleeps with it. + */ +int +i915_wait_request(struct drm_i915_private *dev_priv, uint32_t seqno, + int interruptible) +{ + int ret = 0; + + KASSERT(seqno != dev_priv->mm.next_gem_seqno); + + /* Check first because poking a wedged chip is bad. */ + if (dev_priv->mm.wedged) + return (EIO); + + if (seqno == dev_priv->mm.next_gem_seqno) { + seqno = i915_add_request(dev_priv); + if (seqno == 0) + return (ENOMEM); + } + + if (!i915_seqno_passed(i915_get_gem_seqno(dev_priv), seqno)) { + mtx_enter(&dev_priv->user_irq_lock); + i915_user_irq_get(dev_priv); + while (ret == 0) { + if (i915_seqno_passed(i915_get_gem_seqno(dev_priv), + seqno) || dev_priv->mm.wedged) + break; + ret = msleep(dev_priv, &dev_priv->user_irq_lock, + PZERO | (interruptible ? PCATCH : 0), "gemwt", 0); + } + i915_user_irq_put(dev_priv); + mtx_leave(&dev_priv->user_irq_lock); + } + if (dev_priv->mm.wedged) + ret = EIO; + + /* Directly dispatch request retiring. While we have the work queue + * to handle this, the waiter on a request often wants an associated + * buffer to have made it to the inactive list, and we would need + * a separate wait queue to handle that. + */ + if (ret == 0) + i915_gem_retire_requests(dev_priv); + + return (ret); +} + +/* + * flush and invalidate the provided domains + * if we have successfully queued a gpu flush, then we return a seqno from + * the request. else (failed or just cpu flushed) we return 0. + */ +u_int32_t +i915_gem_flush(struct drm_i915_private *dev_priv, uint32_t invalidate_domains, + uint32_t flush_domains) +{ + uint32_t cmd; + + if (flush_domains & I915_GEM_DOMAIN_CPU) + inteldrm_chipset_flush(dev_priv); + if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) { + /* + * read/write caches: + * + * I915_GEM_DOMAIN_RENDER is always invalidated, but is + * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is + * also flushed at 2d versus 3d pipeline switches. + * + * read-only caches: + * + * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if + * MI_READ_FLUSH is set, and is always flushed on 965. + * + * I915_GEM_DOMAIN_COMMAND may not exist? + * + * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is + * invalidated when MI_EXE_FLUSH is set. + * + * I915_GEM_DOMAIN_VERTEX, which exists on 965, is + * invalidated with every MI_FLUSH. + * + * TLBs: + * + * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND + * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and + * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER + * are flushed at any MI_FLUSH. + */ + + cmd = MI_FLUSH | MI_NO_WRITE_FLUSH; + if ((invalidate_domains | flush_domains) & + I915_GEM_DOMAIN_RENDER) + cmd &= ~MI_NO_WRITE_FLUSH; + /* + * On the 965, the sampler cache always gets flushed + * and this bit is reserved. + */ + if (!IS_I965G(dev_priv) && + invalidate_domains & I915_GEM_DOMAIN_SAMPLER) + cmd |= MI_READ_FLUSH; + if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION) + cmd |= MI_EXE_FLUSH; + + BEGIN_LP_RING(2); + OUT_RING(cmd); + OUT_RING(MI_NOOP); + ADVANCE_LP_RING(); + } + + /* if this is a gpu flush, process the results */ + if (flush_domains & I915_GEM_GPU_DOMAINS) { + inteldrm_process_flushing(dev_priv, flush_domains); + return (i915_add_request(dev_priv)); + } + + return (0); +} + +/** + * Unbinds an object from the GTT aperture. + * + * XXX track dirty and pass down to uvm (note, DONTNEED buffers are clean). + */ +int +i915_gem_object_unbind(struct drm_obj *obj, int interruptible) +{ + struct drm_device *dev = obj->dev; + drm_i915_private_t *dev_priv = dev->dev_private; + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + int ret = 0; + + /* + * if it's already unbound, or we've already done lastclose, just + * let it happen. XXX does this fail to unwire? + */ + if (obj_priv->dmamap == NULL || dev_priv->agpdmat == NULL) + return 0; + + if (obj_priv->pin_count != 0) { + DRM_ERROR("Attempting to unbind pinned buffer\n"); + return (EINVAL); + } + + KASSERT(!i915_obj_purged(obj_priv)); + + /* Move the object to the CPU domain to ensure that + * any possible CPU writes while it's not in the GTT + * are flushed when we go to remap it. This will + * also ensure that all pending GPU writes are finished + * before we unbind. + */ + ret = i915_gem_object_set_to_cpu_domain(obj, 1, interruptible); + if (ret) + return ret; + + KASSERT(!inteldrm_is_active(obj_priv)); + + /* if it's purgeable don't bother dirtying the pages */ + if (i915_obj_purgeable(obj_priv)) + atomic_clearbits_int(&obj_priv->io_flags, I915_DIRTY); + /* + * unload the map, then unwire the backing object. + */ + i915_gem_save_bit_17_swizzle(obj); + bus_dmamap_unload(dev_priv->agpdmat, obj_priv->dmamap); + uvm_objunwire(obj->uao, 0, obj->size); + /* XXX persistent dmamap worth the memory? */ + bus_dmamap_destroy(dev_priv->agpdmat, obj_priv->dmamap); + obj_priv->dmamap = NULL; + free(obj_priv->dma_segs, M_DRM); + obj_priv->dma_segs = NULL; + /* XXX this should change whether we tell uvm the page is dirty */ + atomic_clearbits_int(&obj_priv->io_flags, I915_DIRTY); + + obj_priv->gtt_offset = 0; + atomic_dec(&dev->gtt_count); + atomic_sub(obj->size, &dev->gtt_memory); + + /* Remove ourselves from any LRU list if present. */ + i915_list_remove((struct inteldrm_obj *)obj); + + if (i915_obj_purgeable(obj_priv)) + inteldrm_purge_obj(obj); + + return (0); +} + +int +i915_gem_evict_something(struct drm_i915_private *dev_priv, size_t min_size, + int interruptible) +{ + struct drm_obj *obj; + struct inteldrm_request *request; + struct inteldrm_obj *obj_priv; + int ret = 0; + + for (;;) { + i915_gem_retire_requests(dev_priv); + + /* If there's an inactive buffer available now, grab it + * and be done. + */ + obj = i915_gem_find_inactive_object(dev_priv, min_size); + if (obj != NULL) { + obj_priv = (struct inteldrm_obj *)obj; + + KASSERT(obj_priv->pin_count == 0); + KASSERT(!inteldrm_is_active(obj_priv)); + + /* Wait on the rendering and unbind the buffer. */ + return (i915_gem_object_unbind(obj, interruptible)); + } + + /* If we didn't get anything, but the ring is still processing + * things, wait for one of those things to finish and hopefully + * leave us a buffer to evict. + */ + if ((request = TAILQ_FIRST(&dev_priv->mm.request_list)) + != NULL) { + ret = i915_wait_request(dev_priv, request->seqno, + interruptible); + if (ret) + return (ret); + + continue; + } + + /* If we didn't have anything on the request list but there + * are buffers awaiting a flush, emit one and try again. + * When we wait on it, those buffers waiting for that flush + * will get moved to inactive. + */ + TAILQ_FOREACH(obj_priv, &dev_priv->mm.flushing_list, list) { + obj = &obj_priv->obj; + if (obj->size >= min_size) + break; + obj = NULL; + } + + if (obj != NULL) { + if (i915_gem_flush(dev_priv, obj->write_domain, + obj->write_domain) == 0) + return (ENOMEM); + continue; + } + + /* + * If we didn't do any of the above, there's no single buffer + * large enough to swap out for the new one, so just evict + * everything and start again. (This should be rare.) + */ + if (!TAILQ_EMPTY(&dev_priv->mm.inactive_list)) + return (i915_gem_evict_inactive(dev_priv)); + else + return (i915_gem_evict_everything(dev_priv, + interruptible)); + } + /* NOTREACHED */ +} + +struct drm_obj * +i915_gem_find_inactive_object(struct drm_i915_private *dev_priv, + size_t min_size) +{ + struct drm_obj *obj, *best = NULL, *first = NULL; + struct inteldrm_obj *obj_priv; + + TAILQ_FOREACH(obj_priv, &dev_priv->mm.inactive_list, list) { + obj = &obj_priv->obj; + if (obj->size >= min_size) { + if ((!inteldrm_is_dirty(obj_priv) || + i915_obj_purgeable(obj_priv)) && + (best == NULL || obj->size < best->size)) { + best = obj; + if (best->size == min_size) + return (best); + } + } + if (first == NULL) + first = obj; + } + + return ((best != NULL) ? best : first); +} + +int +i915_gem_evict_everything(struct drm_i915_private *dev_priv, int interruptible) +{ + u_int32_t seqno; + int ret; + + if (TAILQ_EMPTY(&dev_priv->mm.inactive_list) && + TAILQ_EMPTY(&dev_priv->mm.flushing_list) && + TAILQ_EMPTY(&dev_priv->mm.active_list)) + return (ENOSPC); + + seqno = i915_gem_flush(dev_priv, I915_GEM_GPU_DOMAINS, + I915_GEM_GPU_DOMAINS); + if (seqno == 0) + return (ENOMEM); + + if ((ret = i915_wait_request(dev_priv, seqno, interruptible)) != 0 || + (ret = i915_gem_evict_inactive(dev_priv)) != 0) + return (ret); + + /* + * All lists should be empty because we flushed the whole queue, then + * we evicted the whole shebang, only pinned objects are still bound. + */ + KASSERT(TAILQ_EMPTY(&dev_priv->mm.inactive_list)); + KASSERT(TAILQ_EMPTY(&dev_priv->mm.flushing_list)); + KASSERT(TAILQ_EMPTY(&dev_priv->mm.active_list)); + + return (0); +} +/* + * return required GTT alignment for an object, taking into account potential + * fence register needs + */ +bus_size_t +i915_gem_get_gtt_alignment(struct drm_obj *obj) +{ + struct drm_device *dev = obj->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + bus_size_t start, i; + + /* + * Minimum alignment is 4k (GTT page size), but fence registers may + * modify this + */ + if (IS_I965G(dev_priv) || obj_priv->tiling_mode == I915_TILING_NONE) + return (4096); + + /* + * Older chips need to be aligned to the size of the smallest fence + * register that can contain the object. + */ + if (IS_I9XX(dev_priv)) + start = 1024 * 1024; + else + start = 512 * 1024; + + for (i = start; i < obj->size; i <<= 1) + ; + + return (i); +} + +void +i965_write_fence_reg(struct inteldrm_fence *reg) +{ + struct drm_obj *obj = reg->obj; + struct drm_device *dev = obj->dev; + drm_i915_private_t *dev_priv = dev->dev_private; + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + int regnum = obj_priv->fence_reg; + u_int64_t val; + + val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) & + 0xfffff000) << 32; + val |= obj_priv->gtt_offset & 0xfffff000; + val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; + if (obj_priv->tiling_mode == I915_TILING_Y) + val |= 1 << I965_FENCE_TILING_Y_SHIFT; + val |= I965_FENCE_REG_VALID; + + I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val); +} + +void +i915_write_fence_reg(struct inteldrm_fence *reg) +{ + struct drm_obj *obj = reg->obj; + struct drm_device *dev = obj->dev; + drm_i915_private_t *dev_priv = dev->dev_private; + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + bus_size_t fence_reg; + u_int32_t val; + u_int32_t pitch_val; + int regnum = obj_priv->fence_reg; + int tile_width; + + if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) || + (obj_priv->gtt_offset & (obj->size - 1))) { + DRM_ERROR("%s: object 0x%08x not 1M or size (0x%zx) aligned\n", + __func__, obj_priv->gtt_offset, obj->size); + return; + } + + if (obj_priv->tiling_mode == I915_TILING_Y && + HAS_128_BYTE_Y_TILING(dev_priv)) + tile_width = 128; + else + tile_width = 512; + + /* Note: pitch better be a power of two tile widths */ + pitch_val = obj_priv->stride / tile_width; + pitch_val = ffs(pitch_val) - 1; + + val = obj_priv->gtt_offset; + if (obj_priv->tiling_mode == I915_TILING_Y) + val |= 1 << I830_FENCE_TILING_Y_SHIFT; + val |= I915_FENCE_SIZE_BITS(obj->size); + val |= pitch_val << I830_FENCE_PITCH_SHIFT; + val |= I830_FENCE_REG_VALID; + + if (regnum < 8) + fence_reg = FENCE_REG_830_0 + (regnum * 4); + else + fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4); + I915_WRITE(fence_reg, val); +} + +void +i830_write_fence_reg(struct inteldrm_fence *reg) +{ + struct drm_obj *obj = reg->obj; + struct drm_device *dev = obj->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + int regnum = obj_priv->fence_reg; + u_int32_t pitch_val, val; + + if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) || + (obj_priv->gtt_offset & (obj->size - 1))) { + DRM_ERROR("object 0x%08x not 512K or size aligned\n", + obj_priv->gtt_offset); + return; + } + + pitch_val = ffs(obj_priv->stride / 128) - 1; + + val = obj_priv->gtt_offset; + if (obj_priv->tiling_mode == I915_TILING_Y) + val |= 1 << I830_FENCE_TILING_Y_SHIFT; + val |= I830_FENCE_SIZE_BITS(obj->size); + val |= pitch_val << I830_FENCE_PITCH_SHIFT; + val |= I830_FENCE_REG_VALID; + + I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val); + +} + +/* + * i915_gem_get_fence_reg - set up a fence reg for an object + * + * When mapping objects through the GTT, userspace wants to be able to write + * to them without having to worry about swizzling if the object is tiled. + * + * This function walks the fence regs looking for a free one, stealing one + * if it can't find any. + * + * It then sets up the reg based on the object's properties: address, pitch + * and tiling format. + */ +int +i915_gem_get_fence_reg(struct drm_obj *obj, int interruptible) +{ + struct drm_device *dev = obj->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + struct inteldrm_obj *old_obj_priv = NULL; + struct drm_obj *old_obj = NULL; + struct inteldrm_fence *reg = NULL; + int i, ret, avail; + + /* If our fence is getting used, just update our place in the LRU */ + if (obj_priv->fence_reg != I915_FENCE_REG_NONE) { + reg = &dev_priv->fence_regs[obj_priv->fence_reg]; + + TAILQ_REMOVE(&dev_priv->mm.fence_list, reg, list); + TAILQ_INSERT_TAIL(&dev_priv->mm.fence_list, reg, list); + return (0); + } + + switch (obj_priv->tiling_mode) { + case I915_TILING_NONE: + DRM_ERROR("allocating a fence for non-tiled object?\n"); + break; + case I915_TILING_X: + if (obj_priv->stride == 0) + return (EINVAL); + if (obj_priv->stride & (512 - 1)) + DRM_ERROR("object 0x%08x is X tiled but has non-512B" + " pitch\n", obj_priv->gtt_offset); + break; + case I915_TILING_Y: + if (obj_priv->stride == 0) + return (EINVAL); + if (obj_priv->stride & (128 - 1)) + DRM_ERROR("object 0x%08x is Y tiled but has non-128B" + " pitch\n", obj_priv->gtt_offset); + break; + } + + /* First try to find a free reg */ + avail = 0; + for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { + reg = &dev_priv->fence_regs[i]; + if (reg->obj == NULL) + break; + + old_obj_priv = (struct inteldrm_obj *)reg->obj; + if (old_obj_priv->pin_count == 0) + avail++; + } + + /* None available, try to steal one or wait for a user to finish */ + if (i == dev_priv->num_fence_regs) { + if (avail == 0) + return (ENOMEM); + + TAILQ_FOREACH(reg, &dev_priv->mm.fence_list, + list) { + old_obj = reg->obj; + old_obj_priv = (struct inteldrm_obj *)old_obj; + + if (old_obj_priv->pin_count) + continue; + + /* Ref it so that wait_rendering doesn't free it under + * us. + */ + drm_gem_object_reference(old_obj); + + break; + } + + i = old_obj_priv->fence_reg; + reg = &dev_priv->fence_regs[i]; + + ret = i915_gem_object_put_fence_reg(old_obj, interruptible); + drm_gem_object_unreference(old_obj); + if (ret != 0) + return (ret); + } + + obj_priv->fence_reg = i; + reg->obj = obj; + TAILQ_INSERT_TAIL(&dev_priv->mm.fence_list, reg, list); + + if (IS_I965G(dev_priv)) + i965_write_fence_reg(reg); + else if (IS_I9XX(dev_priv)) + i915_write_fence_reg(reg); + else + i830_write_fence_reg(reg); + + return 0; +} + +int +i915_gem_object_put_fence_reg(struct drm_obj *obj, int interruptible) +{ + struct drm_device *dev = obj->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + struct inteldrm_fence *reg; + int ret; + + if (obj_priv->fence_reg == I915_FENCE_REG_NONE) + return (0); + + /* + * If the last execbuffer we did on the object needed a fence then + * we must emit a flush. + */ + if (inteldrm_needs_fence(obj_priv)) { + ret = i915_gem_object_flush_gpu_write_domain(obj, 1, + interruptible); + if (ret != 0) + return (ret); + } + + /* if rendering is queued up that depends on the fence, wait for it */ + reg = &dev_priv->fence_regs[obj_priv->fence_reg]; + if (reg->last_rendering_seqno != 0) { + ret = i915_wait_request(dev_priv, reg->last_rendering_seqno, + interruptible); + if (ret != 0) + return (ret); + } + + /* tiling changed, must wipe userspace mappings */ + if ((obj->write_domain | obj->read_domains) & I915_GEM_DOMAIN_GTT) { + inteldrm_wipe_mappings(obj); + if (obj->write_domain == I915_GEM_DOMAIN_GTT) + obj->write_domain = 0; + } + + if (IS_I965G(dev_priv)) { + I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0); + } else { + u_int32_t fence_reg; + + if (obj_priv->fence_reg < 8) + fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4; + else + fence_reg = FENCE_REG_945_8 + + (obj_priv->fence_reg - 8) * 4; + I915_WRITE(fence_reg , 0); + } + + reg->obj = NULL; + TAILQ_REMOVE(&dev_priv->mm.fence_list, reg, list); + obj_priv->fence_reg = I915_FENCE_REG_NONE; + + return (0); +} + +int +inteldrm_fault(struct drm_obj *obj, struct uvm_faultinfo *ufi, off_t offset, + vaddr_t vaddr, vm_page_t *pps, int npages, int centeridx, + vm_prot_t access_type, int flags) +{ + struct drm_device *dev = obj->dev; + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + paddr_t paddr; + int lcv, ret; + int write = !!(access_type & VM_PROT_WRITE); + vm_prot_t mapprot; + + DRM_LOCK(); + /* + * XXX is it ok to sleep in fault handlers? If not, we may have some + * problems... (i can has race plz? -> judicious use of + * uvmfault_unlockall ahoy) + */ + if (obj_priv->dmamap == NULL) { + ret = i915_gem_object_bind_to_gtt(obj, 0, 0); + if (ret) { + printf("%s: failed to bind\n", __func__); + goto error; + } + i915_gem_object_move_to_inactive(obj); + } + + /* + * We could only do this on bind so allow for map_buffer_range + * unsynchronised objects (where buffer suballocation + * is done by the GL application, however it gives coherency problems + * normally. + */ + ret = i915_gem_object_set_to_gtt_domain(obj, write, 0); + if (ret) { + printf("%s: failed to set to gtt (%d)\n", + __func__, ret); + goto error; + } + + mapprot = ufi->entry->protection; + /* + * if it's only a read fault, we only put ourselves into the gtt + * read domain, so make sure we fault again and set ourselves to write. + * this prevents us needing userland to do domain management and get + * it wrong, and makes us fully coherent with the gpu re mmap. + */ + if (write == 0) + mapprot &= ~VM_PROT_WRITE; + /* XXX try and be more efficient when we do this */ + for (lcv = 0 ; lcv < npages ; lcv++, offset += PAGE_SIZE, + vaddr += PAGE_SIZE) { + if ((flags & PGO_ALLPAGES) == 0 && lcv != centeridx) + continue; + + if (pps[lcv] == PGO_DONTCARE) + continue; + + paddr = dev->agp->base + obj_priv->gtt_offset + offset; + + UVMHIST_LOG(maphist, + " MAPPING: device: pm=%p, va=0x%lx, pa=0x%lx, at=%ld", + ufi->orig_map->pmap, vaddr, (u_long)paddr, mapprot); + /* XXX writecombining */ + if (pmap_enter(ufi->orig_map->pmap, vaddr, paddr | PMAP_NOCACHE, + mapprot, PMAP_CANFAIL | mapprot) != 0) { + DRM_UNLOCK(); + printf("%s: enter failed\n", __func__); + return (VM_PAGER_REFAULT); + } + } + DRM_UNLOCK(); + return (VM_PAGER_OK); + +error: + /* + * EIO means we're wedged so when we reset the gpu this will + * work, so don't segfault. + */ + DRM_UNLOCK(); + return ((ret == EIO) ? VM_PAGER_REFAULT : VM_PAGER_ERROR); + +} + +void +inteldrm_wipe_mappings(struct drm_obj *obj) +{ + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + struct drm_device *dev = obj->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct vm_page *pg; + + /* nuke all our mappings. XXX optimise. */ + for (pg = &dev_priv->pgs[atop(obj_priv->gtt_offset)]; pg != + &dev_priv->pgs[atop(obj_priv->gtt_offset + obj->size)]; pg++) + pmap_page_protect(pg, VM_PROT_NONE); +} + +/** + * Finds free space in the GTT aperture and binds the object there. + */ +int +i915_gem_object_bind_to_gtt(struct drm_obj *obj, bus_size_t alignment, + int interruptible) +{ + struct drm_device *dev = obj->dev; + drm_i915_private_t *dev_priv = dev->dev_private; + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + int ret; + + if (dev_priv->agpdmat == NULL) + return (EINVAL); + if (alignment == 0) { + alignment = i915_gem_get_gtt_alignment(obj); + } else if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) { + DRM_ERROR("Invalid object alignment requested %u\n", alignment); + return (EINVAL); + } + + /* Can't bind a purgeable buffer */ + if (i915_obj_purgeable(obj_priv)) { + printf("tried to bind purgeable buffer"); + return (EINVAL); + } + + if ((ret = bus_dmamap_create(dev_priv->agpdmat, obj->size, 1, + obj->size, 0, BUS_DMA_WAITOK, &obj_priv->dmamap)) != 0) { + DRM_ERROR("Failed to create dmamap\n"); + return (ret); + } + agp_bus_dma_set_alignment(dev_priv->agpdmat, obj_priv->dmamap, + alignment); + + search_free: + /* + * the helper function wires the uao then binds it to the aperture for + * us, so all we have to do is set up the dmamap then load it. + */ + ret = drm_gem_load_uao(dev_priv->agpdmat, obj_priv->dmamap, obj->uao, + obj->size, BUS_DMA_WAITOK | obj_priv->dma_flags, + &obj_priv->dma_segs); + /* XXX NOWAIT? */ + if (ret != 0) { + /* If the gtt is empty and we're still having trouble + * fitting our object in, we're out of memory. + */ + if (TAILQ_EMPTY(&dev_priv->mm.inactive_list) && + TAILQ_EMPTY(&dev_priv->mm.flushing_list) && + TAILQ_EMPTY(&dev_priv->mm.active_list)) { + DRM_ERROR("GTT full, but LRU list empty\n"); + goto error; + } + + ret = i915_gem_evict_something(dev_priv, obj->size, + interruptible); + if (ret != 0) + goto error; + goto search_free; + } + i915_gem_bit_17_swizzle(obj); + + obj_priv->gtt_offset = obj_priv->dmamap->dm_segs[0].ds_addr - + dev->agp->base; + + atomic_inc(&dev->gtt_count); + atomic_add(obj->size, &dev->gtt_memory); + + /* Assert that the object is not currently in any GPU domain. As it + * wasn't in the GTT, there shouldn't be any way it could have been in + * a GPU cache + */ + KASSERT((obj->read_domains & I915_GEM_GPU_DOMAINS) == 0); + KASSERT((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0); + + return (0); + +error: + bus_dmamap_destroy(dev_priv->agpdmat, obj_priv->dmamap); + obj_priv->dmamap = NULL; + obj_priv->gtt_offset = 0; + return (ret); +} + +/* + * Flush the GPU write domain for the object if dirty, then wait for the + * rendering to complete. When this returns it is safe to unbind from the + * GTT or access from the CPU. + */ +int +i915_gem_object_flush_gpu_write_domain(struct drm_obj *obj, int pipelined, + int interruptible) +{ + struct drm_device *dev = obj->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + int ret = 0; + + if ((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0) { + /* + * Queue the GPU write cache flushing we need. + * This call will move stuff form the flushing list to the + * active list so all we need to is wait for it. + */ + (void)i915_gem_flush(dev_priv, 0, obj->write_domain); + KASSERT(obj->write_domain == 0); + } + + /* wait for queued rendering so we know it's flushed and bo is idle */ + if (pipelined == 0 && inteldrm_is_active(obj_priv)) { + ret = i915_wait_request(dev_priv, + obj_priv->last_rendering_seqno, interruptible); + } + return (ret); +} + +/* + * Moves a single object to the GTT and possibly write domain. + * + * This function returns when the move is complete, including waiting on + * flushes to occur. + */ +int +i915_gem_object_set_to_gtt_domain(struct drm_obj *obj, int write, + int interruptible) +{ + struct drm_device *dev = (struct drm_device *)obj->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + int ret; + + /* Not valid to be called on unbound objects. */ + if (obj_priv->dmamap == NULL) + return (EINVAL); + /* Wait on any GPU rendering and flushing to occur. */ + if ((ret = i915_gem_object_flush_gpu_write_domain(obj, 0, + interruptible)) != 0) + return (ret); + + if (obj->write_domain == I915_GEM_DOMAIN_CPU) { + /* clflush the pages, and flush chipset cache */ + bus_dmamap_sync(dev_priv->agpdmat, obj_priv->dmamap, 0, + obj->size, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); + inteldrm_chipset_flush(dev_priv); + obj->write_domain = 0; + } + + /* We're accessing through the gpu, so grab a new fence register or + * update the LRU. + */ + if (obj_priv->tiling_mode != I915_TILING_NONE) + ret = i915_gem_get_fence_reg(obj, interruptible); + + /* + * If we're writing through the GTT domain then the CPU and GPU caches + * will need to be invalidated at next use. + * It should now be out of any other write domains and we can update + * to the correct ones + */ + KASSERT((obj->write_domain & ~I915_GEM_DOMAIN_GTT) == 0); + if (write) { + obj->read_domains = obj->write_domain = I915_GEM_DOMAIN_GTT; + atomic_setbits_int(&obj_priv->io_flags, I915_DIRTY); + } else { + obj->read_domains |= I915_GEM_DOMAIN_GTT; + } + + return (ret); +} + +/* + * Moves a single object to the CPU read and possibly write domain. + * + * This function returns when the move is complete, including waiting on + * flushes to return. + */ +int +i915_gem_object_set_to_cpu_domain(struct drm_obj *obj, int write, + int interruptible) +{ + struct drm_device *dev = obj->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + int ret; + + /* Wait on any GPU rendering and flushing to occur. */ + if ((ret = i915_gem_object_flush_gpu_write_domain(obj, 0, + interruptible)) != 0) + return (ret); + + if (obj->write_domain == I915_GEM_DOMAIN_GTT || + (write && obj->read_domains & I915_GEM_DOMAIN_GTT)) { + /* + * No actual flushing is required for the GTT write domain. + * Writes to it immeditately go to main memory as far as we + * know, so there's no chipset flush. It also doesn't land + * in render cache. + */ + inteldrm_wipe_mappings(obj); + if (obj->write_domain == I915_GEM_DOMAIN_GTT) + obj->write_domain = 0; + } + + /* remove the fence register since we're not using it anymore */ + if ((ret = i915_gem_object_put_fence_reg(obj, interruptible)) != 0) + return (ret); + + /* Flush the CPU cache if it's still invalid. */ + if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) { + bus_dmamap_sync(dev_priv->agpdmat, obj_priv->dmamap, 0, + obj->size, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + + obj->read_domains |= I915_GEM_DOMAIN_CPU; + } + + /* + * It should now be out of any other write domain, and we can update + * the domain value for our changes. + */ + KASSERT((obj->write_domain & ~I915_GEM_DOMAIN_CPU) == 0); + + /* + * If we're writing through the CPU, then the GPU read domains will + * need to be invalidated at next use. + */ + if (write) + obj->read_domains = obj->write_domain = I915_GEM_DOMAIN_CPU; + + return (0); +} + +/* + * Set the next domain for the specified object. This + * may not actually perform the necessary flushing/invaliding though, + * as that may want to be batched with other set_domain operations + * + * This is (we hope) the only really tricky part of gem. The goal + * is fairly simple -- track which caches hold bits of the object + * and make sure they remain coherent. A few concrete examples may + * help to explain how it works. For shorthand, we use the notation + * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the + * a pair of read and write domain masks. + * + * Case 1: the batch buffer + * + * 1. Allocated + * 2. Written by CPU + * 3. Mapped to GTT + * 4. Read by GPU + * 5. Unmapped from GTT + * 6. Freed + * + * Let's take these a step at a time + * + * 1. Allocated + * Pages allocated from the kernel may still have + * cache contents, so we set them to (CPU, CPU) always. + * 2. Written by CPU (using pwrite) + * The pwrite function calls set_domain (CPU, CPU) and + * this function does nothing (as nothing changes) + * 3. Mapped by GTT + * This function asserts that the object is not + * currently in any GPU-based read or write domains + * 4. Read by GPU + * i915_gem_execbuffer calls set_domain (COMMAND, 0). + * As write_domain is zero, this function adds in the + * current read domains (CPU+COMMAND, 0). + * flush_domains is set to CPU. + * invalidate_domains is set to COMMAND + * clflush is run to get data out of the CPU caches + * then i915_dev_set_domain calls i915_gem_flush to + * emit an MI_FLUSH and drm_agp_chipset_flush + * 5. Unmapped from GTT + * i915_gem_object_unbind calls set_domain (CPU, CPU) + * flush_domains and invalidate_domains end up both zero + * so no flushing/invalidating happens + * 6. Freed + * yay, done + * + * Case 2: The shared render buffer + * + * 1. Allocated + * 2. Mapped to GTT + * 3. Read/written by GPU + * 4. set_domain to (CPU,CPU) + * 5. Read/written by CPU + * 6. Read/written by GPU + * + * 1. Allocated + * Same as last example, (CPU, CPU) + * 2. Mapped to GTT + * Nothing changes (assertions find that it is not in the GPU) + * 3. Read/written by GPU + * execbuffer calls set_domain (RENDER, RENDER) + * flush_domains gets CPU + * invalidate_domains gets GPU + * clflush (obj) + * MI_FLUSH and drm_agp_chipset_flush + * 4. set_domain (CPU, CPU) + * flush_domains gets GPU + * invalidate_domains gets CPU + * flush_gpu_write (obj) to make sure all drawing is complete. + * This will include an MI_FLUSH to get the data from GPU + * to memory + * clflush (obj) to invalidate the CPU cache + * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?) + * 5. Read/written by CPU + * cache lines are loaded and dirtied + * 6. Read written by GPU + * Same as last GPU access + * + * Case 3: The constant buffer + * + * 1. Allocated + * 2. Written by CPU + * 3. Read by GPU + * 4. Updated (written) by CPU again + * 5. Read by GPU + * + * 1. Allocated + * (CPU, CPU) + * 2. Written by CPU + * (CPU, CPU) + * 3. Read by GPU + * (CPU+RENDER, 0) + * flush_domains = CPU + * invalidate_domains = RENDER + * clflush (obj) + * MI_FLUSH + * drm_agp_chipset_flush + * 4. Updated (written) by CPU again + * (CPU, CPU) + * flush_domains = 0 (no previous write domain) + * invalidate_domains = 0 (no new read domains) + * 5. Read by GPU + * (CPU+RENDER, 0) + * flush_domains = CPU + * invalidate_domains = RENDER + * clflush (obj) + * MI_FLUSH + * drm_agp_chipset_flush + */ +void +i915_gem_object_set_to_gpu_domain(struct drm_obj *obj) +{ + struct drm_device *dev = obj->dev; + drm_i915_private_t *dev_priv = dev->dev_private; + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + u_int32_t invalidate_domains = 0; + u_int32_t flush_domains = 0; + + KASSERT((obj->pending_read_domains & I915_GEM_DOMAIN_CPU) == 0); + KASSERT(obj->pending_write_domain != I915_GEM_DOMAIN_CPU); + /* + * If the object isn't moving to a new write domain, + * let the object stay in multiple read domains + */ + if (obj->pending_write_domain == 0) + obj->pending_read_domains |= obj->read_domains; + else + atomic_setbits_int(&obj_priv->io_flags, I915_DIRTY); + + /* + * Flush the current write domain if + * the new read domains don't match. Invalidate + * any read domains which differ from the old + * write domain + */ + if (obj->write_domain && + obj->write_domain != obj->pending_read_domains) { + flush_domains |= obj->write_domain; + invalidate_domains |= obj->pending_read_domains & + ~obj->write_domain; + } + /* + * Invalidate any read caches which may have + * stale data. That is, any new read domains. + */ + invalidate_domains |= obj->pending_read_domains & ~obj->read_domains; + /* clflush the cpu now, gpu caches get queued. */ + if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) { + bus_dmamap_sync(dev_priv->agpdmat, obj_priv->dmamap, 0, + obj->size, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + } + if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_GTT) { + inteldrm_wipe_mappings(obj); + } + + /* The actual obj->write_domain will be updated with + * pending_write_domain after we emit the accumulated flush for all of + * the domain changes in execuffer (which clears object's write + * domains). So if we have a current write domain that we aren't + * changing, set pending_write_domain to it. + */ + if (flush_domains == 0 && obj->pending_write_domain == 0 && + (obj->pending_read_domains == obj->write_domain || + obj->write_domain == 0)) + obj->pending_write_domain = obj->write_domain; + obj->read_domains = obj->pending_read_domains; + obj->pending_read_domains = 0; + + dev->invalidate_domains |= invalidate_domains; + dev->flush_domains |= flush_domains; +} + +/** + * Pin an object to the GTT and evaluate the relocations landing in it. + */ +int +i915_gem_object_pin_and_relocate(struct drm_obj *obj, + struct drm_file *file_priv, struct drm_i915_gem_exec_object2 *entry, + struct drm_i915_gem_relocation_entry *relocs) +{ + struct drm_device *dev = obj->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_obj *target_obj; + struct inteldrm_obj *obj_priv = + (struct inteldrm_obj *)obj; + bus_space_handle_t bsh; + int i, ret, needs_fence; + + needs_fence = (entry->flags & EXEC_OBJECT_NEEDS_FENCE) && + obj_priv->tiling_mode != I915_TILING_NONE; + if (needs_fence) + atomic_setbits_int(&obj_priv->io_flags, I915_EXEC_NEEDS_FENCE); + + /* Choose the GTT offset for our buffer and put it there. */ + ret = i915_gem_object_pin(obj, (u_int32_t)entry->alignment, + needs_fence); + /* XXX what if already bound at a different alignment? */ + if (ret) + return ret; + + entry->offset = obj_priv->gtt_offset; + + /* Apply the relocations, using the GTT aperture to avoid cache + * flushing requirements. + */ + for (i = 0; i < entry->relocation_count; i++) { + struct drm_i915_gem_relocation_entry *reloc = &relocs[i]; + struct inteldrm_obj *target_obj_priv; + uint32_t reloc_val, reloc_offset; + + target_obj = drm_gem_object_lookup(obj->dev, file_priv, + reloc->target_handle); + if (target_obj == NULL) { + i915_gem_object_unpin(obj); + return (EBADF); + } + target_obj_priv = (struct inteldrm_obj *)target_obj; + + /* The target buffer should have appeared before us in the + * exec_object list, so it should have a GTT space bound by now. + */ + if (target_obj_priv->dmamap == 0) { + DRM_ERROR("No GTT space found for object %d\n", + reloc->target_handle); + ret = EINVAL; + goto err; + } + + /* must be in one write domain and one only */ + if (reloc->write_domain & (reloc->write_domain - 1)) { + ret = EINVAL; + goto err; + } + if (reloc->read_domains & I915_GEM_DOMAIN_CPU || + reloc->write_domain & I915_GEM_DOMAIN_CPU) { + DRM_ERROR("relocation with read/write CPU domains: " + "obj %p target %d offset %d " + "read %08x write %08x", obj, + reloc->target_handle, (int)reloc->offset, + reloc->read_domains, reloc->write_domain); + ret = EINVAL; + goto err; + } + + if (reloc->write_domain && target_obj->pending_write_domain && + reloc->write_domain != target_obj->pending_write_domain) { + DRM_ERROR("Write domain conflict: " + "obj %p target %d offset %d " + "new %08x old %08x\n", + obj, reloc->target_handle, + (int) reloc->offset, + reloc->write_domain, + target_obj->pending_write_domain); + ret = EINVAL; + goto err; + } + + target_obj->pending_read_domains |= reloc->read_domains; + target_obj->pending_write_domain |= reloc->write_domain; + + + if (reloc->offset > obj->size - 4) { + DRM_ERROR("Relocation beyond object bounds: " + "obj %p target %d offset %d size %d.\n", + obj, reloc->target_handle, + (int) reloc->offset, (int) obj->size); + ret = EINVAL; + goto err; + } + if (reloc->offset & 3) { + DRM_ERROR("Relocation not 4-byte aligned: " + "obj %p target %d offset %d.\n", + obj, reloc->target_handle, + (int) reloc->offset); + ret = EINVAL; + goto err; + } + + if (reloc->delta > target_obj->size) { + DRM_ERROR("reloc larger than target\n"); + ret = EINVAL; + goto err; + } + + /* Map the page containing the relocation we're going to + * perform. + */ + reloc_offset = obj_priv->gtt_offset + reloc->offset; + reloc_val = target_obj_priv->gtt_offset + reloc->delta; + + if ((ret = bus_space_subregion(dev_priv->bst, + dev_priv->aperture_bsh, trunc_page(reloc_offset), + PAGE_SIZE, &bsh)) != 0) { + DRM_ERROR("map failed...\n"); + goto err; + } + /* + * we do this differently to linux, in the case where the + * presumed offset matches we actually read to check it's + * correct, but at least it won't involve idling the gpu if + * it was reading from it before, only if writing (which would + * be bad anyway since we're now using it as a command buffer). + */ + if (target_obj_priv->gtt_offset == reloc->presumed_offset) { + ret = i915_gem_object_set_to_gtt_domain(obj, 0, 1); + if (ret != 0) + goto err; + if (bus_space_read_4(dev_priv->bst, bsh, + reloc_offset & PAGE_MASK) == reloc_val) { + drm_gem_object_unreference(target_obj); + continue; + } + DRM_DEBUG("reloc tested and found incorrect\n"); + } + + ret = i915_gem_object_set_to_gtt_domain(obj, 1, 1); + if (ret != 0) + goto err; + + bus_space_write_4(dev_priv->bst, bsh, reloc_offset & PAGE_MASK, + reloc_val); + + reloc->presumed_offset = target_obj_priv->gtt_offset; + + drm_gem_object_unreference(target_obj); + } + + return 0; + +err: + /* we always jump to here mid-loop */ + drm_gem_object_unreference(target_obj); + i915_gem_object_unpin(obj); + return (ret); + +} + +/** Dispatch a batchbuffer to the ring + */ +void +i915_dispatch_gem_execbuffer(struct drm_device *dev, + struct drm_i915_gem_execbuffer2 *exec, uint64_t exec_offset) +{ + drm_i915_private_t *dev_priv = dev->dev_private; + uint32_t exec_start, exec_len; + + exec_start = (uint32_t)exec_offset + exec->batch_start_offset; + exec_len = (uint32_t)exec->batch_len; + + if (IS_I830(dev_priv) || IS_845G(dev_priv)) { + BEGIN_LP_RING(6); + OUT_RING(MI_BATCH_BUFFER); + OUT_RING(exec_start | MI_BATCH_NON_SECURE); + OUT_RING(exec_start + exec_len - 4); + OUT_RING(MI_NOOP); + } else { + BEGIN_LP_RING(4); + if (IS_I965G(dev_priv)) { + OUT_RING(MI_BATCH_BUFFER_START | (2 << 6) | + MI_BATCH_NON_SECURE_I965); + OUT_RING(exec_start); + } else { + OUT_RING(MI_BATCH_BUFFER_START | (2 << 6)); + OUT_RING(exec_start | MI_BATCH_NON_SECURE); + } + } + + /* + * Ensure that the commands in the batch buffer are + * finished before the interrupt fires (from a subsequent request + * added). We get back a seqno representing the execution of the + * current buffer, which we can wait on. We would like to mitigate + * these interrupts, likely by only creating seqnos occasionally + * (so that we have *some* interrupts representing completion of + * buffers that we can wait on when trying to clear up gtt space). + */ + OUT_RING(MI_FLUSH | MI_NO_WRITE_FLUSH); + OUT_RING(MI_NOOP); + ADVANCE_LP_RING(); + + inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__); + /* The sampler always gets flushed on i965 (sigh) */ + if (IS_I965G(dev_priv)) + inteldrm_process_flushing(dev_priv, I915_GEM_DOMAIN_SAMPLER); +} + +/* Throttle our rendering by waiting until the ring has completed our requests + * emitted over 20 msec ago. + * + * This should get us reasonable parallelism between CPU and GPU but also + * relatively low latency when blocking on a particular request to finish. + */ +int +i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv) +{ +#if 0 + struct inteldrm_file *intel_file = (struct inteldrm_file *)file_priv; + u_int32_t seqno; +#endif + int ret = 0; + + return ret; +} + +int +i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list, + u_int32_t buffer_count, struct drm_i915_gem_relocation_entry **relocs) +{ + u_int32_t reloc_count = 0, reloc_index = 0, i; + int ret; + + *relocs = NULL; + for (i = 0; i < buffer_count; i++) { + if (reloc_count + exec_list[i].relocation_count < reloc_count) + return (EINVAL); + reloc_count += exec_list[i].relocation_count; + } + + if (reloc_count == 0) + return (0); + + if (SIZE_MAX / reloc_count < sizeof(**relocs)) + return (EINVAL); + *relocs = drm_alloc(reloc_count * sizeof(**relocs)); + for (i = 0; i < buffer_count; i++) { + if ((ret = copyin((void *)(uintptr_t)exec_list[i].relocs_ptr, + &(*relocs)[reloc_index], exec_list[i].relocation_count * + sizeof(**relocs))) != 0) { + drm_free(*relocs); + *relocs = NULL; + return (ret); + } + reloc_index += exec_list[i].relocation_count; + } + + return (0); +} + +int +i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list, + u_int32_t buffer_count, struct drm_i915_gem_relocation_entry *relocs) +{ + u_int32_t reloc_count = 0, i; + int ret = 0; + + if (relocs == NULL) + return (0); + + for (i = 0; i < buffer_count; i++) { + if ((ret = copyout(&relocs[reloc_count], + (void *)(uintptr_t)exec_list[i].relocs_ptr, + exec_list[i].relocation_count * sizeof(*relocs))) != 0) + break; + reloc_count += exec_list[i].relocation_count; + } + + drm_free(relocs); + + return (ret); +} + +int +i915_gem_execbuffer2(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_gem_execbuffer2 *args = data; + struct drm_i915_gem_exec_object2 *exec_list = NULL; + struct drm_i915_gem_relocation_entry *relocs; + struct inteldrm_obj *obj_priv, *batch_obj_priv; + struct drm_obj **object_list = NULL; + struct drm_obj *batch_obj, *obj; + size_t oflow; + int ret, ret2, i; + int pinned = 0, pin_tries; + uint32_t reloc_index; + + /* + * Check for valid execbuffer offset. We can do this early because + * bound object are always page aligned, so only the start offset + * matters. Also check for integer overflow in the batch offset and size + */ + if ((args->batch_start_offset | args->batch_len) & 0x7 || + args->batch_start_offset + args->batch_len < args->batch_len || + args->batch_start_offset + args->batch_len < + args->batch_start_offset) + return (EINVAL); + + if (args->buffer_count < 1) { + DRM_ERROR("execbuf with %d buffers\n", args->buffer_count); + return (EINVAL); + } + /* Copy in the exec list from userland, check for overflow */ + oflow = SIZE_MAX / args->buffer_count; + if (oflow < sizeof(*exec_list) || oflow < sizeof(*object_list)) + return (EINVAL); + exec_list = drm_alloc(sizeof(*exec_list) * args->buffer_count); + object_list = drm_alloc(sizeof(*object_list) * args->buffer_count); + if (exec_list == NULL || object_list == NULL) { + ret = ENOMEM; + goto pre_mutex_err; + } + ret = copyin((void *)(uintptr_t)args->buffers_ptr, exec_list, + sizeof(*exec_list) * args->buffer_count); + if (ret != 0) + goto pre_mutex_err; + + ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count, + &relocs); + if (ret != 0) + goto pre_mutex_err; + + DRM_LOCK(); + inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__); + + /* XXX check these before we copyin... but we do need the lock */ + if (dev_priv->mm.wedged) { + ret = EIO; + goto unlock; + } + + if (dev_priv->mm.suspended) { + ret = EBUSY; + goto unlock; + } + + /* Look up object handles */ + for (i = 0; i < args->buffer_count; i++) { + object_list[i] = drm_gem_object_lookup(dev, file_priv, + exec_list[i].handle); + if (object_list[i] == NULL) { + DRM_ERROR("Invalid object handle %d at index %d\n", + exec_list[i].handle, i); + ret = EBADF; + goto err; + } + obj_priv = (struct inteldrm_obj *)object_list[i]; + if (obj_priv->io_flags & I915_IN_EXEC) { + DRM_ERROR("Object %p appears more than once in object_list\n", + object_list[i]); + ret = EBADF; + goto err; + } + atomic_setbits_int(&obj_priv->io_flags, I915_IN_EXEC); + } + + /* Pin and relocate */ + for (pin_tries = 0; ; pin_tries++) { + ret = pinned = 0; + reloc_index = 0; + + for (i = 0; i < args->buffer_count; i++) { + object_list[i]->pending_read_domains = 0; + object_list[i]->pending_write_domain = 0; + ret = i915_gem_object_pin_and_relocate(object_list[i], + file_priv, &exec_list[i], &relocs[reloc_index]); + if (ret) + break; + pinned++; + reloc_index += exec_list[i].relocation_count; + } + /* success */ + if (ret == 0) + break; + + /* error other than GTT full, or we've already tried again */ + if (ret != ENOSPC || pin_tries >= 1) + goto err; + + /* unpin all of our buffers */ + for (i = 0; i < pinned; i++) + i915_gem_object_unpin(object_list[i]); + /* evict everyone we can from the aperture */ + ret = i915_gem_evict_everything(dev_priv, 1); + if (ret) + goto err; + } + + /* Set the pending read domains for the batch buffer to COMMAND */ + batch_obj = object_list[args->buffer_count - 1]; + batch_obj_priv = (struct inteldrm_obj *)batch_obj; + if (args->batch_start_offset + args->batch_len > batch_obj->size || + batch_obj->pending_write_domain) { + ret = EINVAL; + goto err; + } + batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND; + + inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__); + + /* + * Zero the global flush/invalidate flags. These will be modified as + * new domains are computed for each object + */ + dev->invalidate_domains = 0; + dev->flush_domains = 0; + + /* Compute new gpu domains and update invalidate/flush */ + for (i = 0; i < args->buffer_count; i++) + i915_gem_object_set_to_gpu_domain(object_list[i]); + + inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__); + + /* flush and invalidate any domains that need them. */ + (void)i915_gem_flush(dev_priv, dev->invalidate_domains, + dev->flush_domains); + + /* + * update the write domains, and fence/gpu write accounting information. + * Also do the move to active list here. The lazy seqno accounting will + * make sure that they have the correct seqno. If the add_request + * fails, then we will wait for a later batch (or one added on the + * wait), which will waste some time, but if we're that low on memory + * then we could fail in much worse ways. + */ + for (i = 0; i < args->buffer_count; i++) { + obj = object_list[i]; + obj_priv = (struct inteldrm_obj *)obj; + + obj->write_domain = obj->pending_write_domain; + /* + * if we have a write domain, add us to the gpu write list + * else we can remove the bit because it has been flushed. + */ + if (obj_priv->io_flags & I915_GPU_WRITE) + TAILQ_REMOVE(&dev_priv->mm.gpu_write_list, obj_priv, + write_list); + if (obj->write_domain) { + TAILQ_INSERT_TAIL(&dev_priv->mm.gpu_write_list, + obj_priv, write_list); + atomic_setbits_int(&obj_priv->io_flags, I915_GPU_WRITE); + } else { + atomic_clearbits_int(&obj_priv->io_flags, + I915_GPU_WRITE); + } + /* if this batchbuffer needs a fence, then the object is + * counted as fenced exec. else any outstanding fence waits + * will just wait on the fence last_seqno. + */ + if (inteldrm_exec_needs_fence(obj_priv)) { + atomic_setbits_int(&obj_priv->io_flags, + I915_FENCED_EXEC); + } else { + atomic_clearbits_int(&obj_priv->io_flags, + I915_FENCED_EXEC); + } + + i915_gem_object_move_to_active(object_list[i]); + } + + inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__); + + /* Exec the batchbuffer */ + /* + * XXX make sure that this may never fail by preallocating the request. + */ + i915_dispatch_gem_execbuffer(dev, args, batch_obj_priv->gtt_offset); + + /* + * move to active associated all previous buffers with the seqno + * that this call will emit. so we don't need the return. + */ + (void)i915_add_request(dev_priv); + + inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__); + + ret = copyout(exec_list, (void *)(uintptr_t)args->buffers_ptr, + sizeof(*exec_list) * args->buffer_count); + +err: + for (i = 0; i < args->buffer_count; i++) { + if (object_list[i] == NULL) + break; + + obj_priv = (struct inteldrm_obj *)object_list[i]; + if (i < pinned) + i915_gem_object_unpin(object_list[i]); + + atomic_clearbits_int(&obj_priv->io_flags, I915_IN_EXEC | + I915_EXEC_NEEDS_FENCE); + drm_gem_object_unreference(object_list[i]); + } + +unlock: + DRM_UNLOCK(); + +pre_mutex_err: + /* update userlands reloc state. */ + ret2 = i915_gem_put_relocs_to_user(exec_list, + args->buffer_count, relocs); + if (ret2 != 0 && ret == 0) + ret = ret2; + + drm_free(object_list); + drm_free(exec_list); + + return ret; +} + +int +i915_gem_object_pin(struct drm_obj *obj, uint32_t alignment, int needs_fence) +{ + struct drm_device *dev = obj->dev; + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + int ret; + + inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__); + /* + * if already bound, but alignment is unsuitable, unbind so we can + * fix it. Similarly if we have constraints due to fence registers, + * adjust if needed. Note that if we are already pinned we may as well + * fail because whatever depends on this alignment will render poorly + * otherwise, so just fail the pin (with a printf so we can fix a + * wrong userland). + */ + if ((alignment && obj_priv->dmamap != NULL && + obj_priv->gtt_offset & (alignment - 1)) || (needs_fence && + !i915_gem_object_fence_offset_ok(obj, obj_priv->tiling_mode))) { + if (obj_priv->pin_count == 0) { + ret = i915_gem_object_unbind(obj, 1); + if (ret) + return (ret); + } else { + DRM_ERROR("repinning an object with bad alignment\n"); + } + return (EINVAL); + } + + if (obj_priv->dmamap == NULL) { + ret = i915_gem_object_bind_to_gtt(obj, alignment, 1); + if (ret != 0) + return (ret); + } + + /* + * Pre-965 chips may need a fence register set up in order to + * handle tiling properly. GTT mapping may have blown it away so + * restore. + * With execbuf2 support we don't always need it, but if we do grab + * it. + */ + if (needs_fence && obj_priv->tiling_mode != I915_TILING_NONE && + (ret = i915_gem_get_fence_reg(obj, 1)) != 0) + return (ret); + + /* If the object is not active and not pending a flush, + * remove it from the inactive list + */ + if (++obj_priv->pin_count == 1) { + atomic_inc(&dev->pin_count); + atomic_add(obj->size, &dev->pin_memory); + if (!inteldrm_is_active(obj_priv)) + i915_list_remove(obj_priv); + } + inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__); + + return (0); +} + +void +i915_gem_object_unpin(struct drm_obj *obj) +{ + struct drm_device *dev = obj->dev; + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + + inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__); + KASSERT(obj_priv->pin_count >= 1); + KASSERT(obj_priv->dmamap != NULL); + + /* If the object is no longer pinned, and is + * neither active nor being flushed, then stick it on + * the inactive list + */ + if (--obj_priv->pin_count == 0) { + if (!inteldrm_is_active(obj_priv)) + i915_gem_object_move_to_inactive(obj); + atomic_dec(&dev->pin_count); + atomic_sub(obj->size, &dev->pin_memory); + } + inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__); +} + +int +i915_gem_pin_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_i915_gem_pin *args = data; + struct drm_obj *obj; + struct inteldrm_obj *obj_priv; + int ret = 0; + + obj = drm_gem_object_lookup(dev, file_priv, args->handle); + if (obj == NULL) + return (EBADF); + + obj_priv = (struct inteldrm_obj *)obj; + DRM_LOCK(); + if (i915_obj_purgeable(obj_priv)) { + printf("pinning purgeable object\n"); + ret = EINVAL; + goto out; + } + + if (++obj_priv->user_pin_count == 1) { + ret = i915_gem_object_pin(obj, args->alignment, 1); + if (ret != 0) + goto out; + } + + /* XXX - flush the CPU caches for pinned objects + * as the X server doesn't manage domains yet + */ + i915_gem_object_set_to_gtt_domain(obj, 1, 1); + args->offset = obj_priv->gtt_offset; + +out: + drm_gem_object_unreference(obj); + DRM_UNLOCK(); + + return (ret); +} + +int +i915_gem_unpin_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_i915_gem_pin *args = data; + struct inteldrm_obj *obj_priv; + struct drm_obj *obj; + + obj = drm_gem_object_lookup(dev, file_priv, args->handle); + if (obj == NULL) + return (EBADF); + + DRM_LOCK(); + obj_priv = (struct inteldrm_obj *)obj; + if (obj_priv->user_pin_count == 0) { + DRM_UNLOCK(); + return (EINVAL); + } + + if (--obj_priv->user_pin_count == 0) + i915_gem_object_unpin(obj); + + drm_gem_object_unreference(obj); + DRM_UNLOCK(); + return (0); +} + +int +i915_gem_busy_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_gem_busy *args = data; + struct drm_obj *obj; + struct inteldrm_obj *obj_priv; + + obj = drm_gem_object_lookup(dev, file_priv, args->handle); + if (obj == NULL) { + DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n", + args->handle); + return (EBADF); + } + + DRM_LOCK(); + /* + * Update the active list for the hardware's current position. + * otherwise this will only update on a delayed timer or when + * the irq is unmasked. This keeps our working set smaller. + */ + i915_gem_retire_requests(dev_priv); + + obj_priv = (struct inteldrm_obj *)obj; + /* + * Don't count being on the flushing list being done. Otherwise, a + * buffer left on the flushing list but not getting flushed (because + * no one is flushing that domain) won't ever return unbusy and get + * reused by libdrm's bo cache. The other expected consumer of this + * interface, OpenGL's occlusion queries, also specs that the object + * get unbusy "eventually" without any interference. + */ + args->busy = inteldrm_is_active(obj_priv) && + obj_priv->last_rendering_seqno != 0; + + drm_gem_object_unreference(obj); + DRM_UNLOCK(); + return 0; +} + +int +i915_gem_madvise_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_i915_gem_madvise *args = data; + struct drm_obj *obj; + struct inteldrm_obj *obj_priv; + int need, ret = 0; + + switch (args->madv) { + case I915_MADV_DONTNEED: + need = 0; + break; + case I915_MADV_WILLNEED: + need = 1; + break; + default: + return (EINVAL); + } + + obj = drm_gem_object_lookup(dev, file_priv, args->handle); + if (obj == NULL) + return (EBADF); + + obj_priv = (struct inteldrm_obj *)obj; + DRM_LOCK(); + + /* invalid to madvise on a pinned BO */ + if (obj_priv->pin_count) { + ret = EINVAL; + goto out; + } + + if (!i915_obj_purged(obj_priv)) { + if (need) { + atomic_clearbits_int(&obj_priv->io_flags, + I915_DONTNEED); + } else { + atomic_setbits_int(&obj_priv->io_flags, I915_DONTNEED); + } + } + + + /* if the object is no longer bound, discard its backing storage */ + if (i915_obj_purgeable(obj_priv) && obj_priv->dmamap == NULL) + inteldrm_purge_obj(obj); + + args->retained = !i915_obj_purged(obj_priv); + +out: + drm_gem_object_unreference(obj); + DRM_UNLOCK(); + + return (ret); +} + +int +i915_gem_init_object(struct drm_obj *obj) +{ + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + + /* + * We've just allocated pages from the kernel, + * so they've just been written by the CPU with + * zeros. They'll need to be flushed before we + * use them with the GPU. + */ + obj->write_domain = I915_GEM_DOMAIN_CPU; + obj->read_domains = I915_GEM_DOMAIN_CPU; + + /* normal objects don't need special treatment */ + obj_priv->dma_flags = 0; + obj_priv->fence_reg = I915_FENCE_REG_NONE; + + return 0; +} + +/* + * NOTE all object unreferences in this driver need to hold the DRM_LOCK(), + * because if they free they poke around in driver structures. + */ +void +i915_gem_free_object(struct drm_obj *obj) +{ + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + + while (obj_priv->pin_count > 0) + i915_gem_object_unpin(obj); + + i915_gem_object_unbind(obj, 0); + /* XXX dmatag went away? */ +} + +/** Unbinds all objects that are on the given buffer list. */ +int +i915_gem_evict_inactive(struct drm_i915_private *dev_priv) +{ + struct inteldrm_obj *obj_priv; + int ret; + + while ((obj_priv = TAILQ_FIRST(&dev_priv->mm.inactive_list)) != NULL) { + if (obj_priv->pin_count != 0) { + DRM_ERROR("Pinned object in unbind list\n"); + return (EINVAL); + } + + if ((ret = i915_gem_object_unbind(&obj_priv->obj, 1)) != 0) + break; + } + + return (ret); +} + +int +i915_gem_idle(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = (struct drm_device *)dev_priv->drmdev; + int ret; + + DRM_LOCK(); + if (dev_priv->mm.suspended || dev_priv->ring.ring_obj == NULL) { + DRM_UNLOCK(); + return (0); + } + + /* + * If we're wedged, the workq will clear everything, else this will + * empty out the lists for us. + */ + if ((ret = i915_gem_evict_everything(dev_priv, 1)) != 0 && ret != ENOSPC) { + DRM_UNLOCK(); + return (ret); + } + + /* Hack! Don't let anybody do execbuf while we don't control the chip. + * We need to replace this with a semaphore, or something. + */ + dev_priv->mm.suspended = 1; + /* if we hung then the timer alredy fired. */ + timeout_del(&dev_priv->mm.hang_timer); + + inteldrm_update_ring(dev_priv); + i915_gem_cleanup_ringbuffer(dev_priv); + DRM_UNLOCK(); + + /* this should be idle now */ + timeout_del(&dev_priv->mm.retire_timer); + + return 0; +} + +int +i915_gem_init_hws(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = (struct drm_device *)dev_priv->drmdev; + struct drm_obj *obj; + struct inteldrm_obj *obj_priv; + int ret; + + /* If we need a physical address for the status page, it's already + * initialized at driver load time. + */ + if (!I915_NEED_GFX_HWS(dev_priv)) + return 0; + + obj = drm_gem_object_alloc(dev, 4096); + if (obj == NULL) { + DRM_ERROR("Failed to allocate status page\n"); + return (ENOMEM); + } + obj_priv = (struct inteldrm_obj *)obj; + /* + * snooped gtt mapping please . + * Normally this flag is only to dmamem_map, but it's been overloaded + * for the agp mapping + */ + obj_priv->dma_flags = BUS_DMA_COHERENT | BUS_DMA_READ; + + ret = i915_gem_object_pin(obj, 4096, 0); + if (ret != 0) { + drm_gem_object_unreference(obj); + return ret; + } + + dev_priv->hw_status_page = (void *)vm_map_min(kernel_map); + obj->uao->pgops->pgo_reference(obj->uao); + if ((ret = uvm_map(kernel_map, (vaddr_t *)&dev_priv->hw_status_page, + PAGE_SIZE, obj->uao, 0, 0, UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW, + UVM_INH_SHARE, UVM_ADV_RANDOM, 0))) != 0) + if (ret != 0) { + DRM_ERROR("Failed to map status page.\n"); + obj->uao->pgops->pgo_detach(obj->uao); + memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map)); + i915_gem_object_unpin(obj); + drm_gem_object_unreference(obj); + return (EINVAL); + } + dev_priv->hws_obj = obj; + memset(dev_priv->hw_status_page, 0, PAGE_SIZE); + I915_WRITE(HWS_PGA, obj_priv->gtt_offset); + I915_READ(HWS_PGA); /* posting read */ + DRM_DEBUG("hws offset: 0x%08x\n", obj_priv->gtt_offset); + + return 0; +} + +void +i915_gem_cleanup_hws(struct drm_i915_private *dev_priv) +{ + struct drm_obj *obj; + struct inteldrm_obj *obj_priv; + + if (dev_priv->hws_obj == NULL) + return; + + obj = dev_priv->hws_obj; + obj_priv = (struct inteldrm_obj *)obj; + + uvm_unmap(kernel_map, (vaddr_t)dev_priv->hw_status_page, + (vaddr_t)dev_priv->hw_status_page + PAGE_SIZE); + i915_gem_object_unpin(obj); + drm_gem_object_unreference(obj); + dev_priv->hws_obj = NULL; + + memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map)); + dev_priv->hw_status_page = NULL; + + /* Write high address into HWS_PGA when disabling. */ + I915_WRITE(HWS_PGA, 0x1ffff000); +} + +int +i915_gem_init_ringbuffer(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = (struct drm_device *)dev_priv->drmdev; + struct drm_obj *obj; + struct inteldrm_obj *obj_priv; + int ret; + + ret = i915_gem_init_hws(dev_priv); + if (ret != 0) + return ret; + + obj = drm_gem_object_alloc(dev, 128 * 1024); + if (obj == NULL) { + DRM_ERROR("Failed to allocate ringbuffer\n"); + ret = ENOMEM; + goto delhws; + } + obj_priv = (struct inteldrm_obj *)obj; + + ret = i915_gem_object_pin(obj, 4096, 0); + if (ret != 0) + goto unref; + + /* Set up the kernel mapping for the ring. */ + dev_priv->ring.size = obj->size; + + /* XXX WC */ + if ((ret = bus_space_subregion(dev_priv->bst, dev_priv->aperture_bsh, + obj_priv->gtt_offset, obj->size, &dev_priv->ring.bsh)) != 0) { + DRM_INFO("can't map ringbuffer\n"); + goto unpin; + } + dev_priv->ring.ring_obj = obj; + + if ((ret = inteldrm_start_ring(dev_priv)) != 0) + goto unmap; + + return (0); + +unmap: +unpin: + memset(&dev_priv->ring, 0, sizeof(dev_priv->ring)); + i915_gem_object_unpin(obj); +unref: + drm_gem_object_unreference(obj); +delhws: + i915_gem_cleanup_hws(dev_priv); + return (ret); +} + +int +inteldrm_start_ring(struct drm_i915_private *dev_priv) +{ + struct drm_obj *obj = dev_priv->ring.ring_obj; + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + u_int32_t head; + + /* Stop the ring if it's running. */ + I915_WRITE(PRB0_CTL, 0); + I915_WRITE(PRB0_TAIL, 0); + I915_WRITE(PRB0_HEAD, 0); + + /* Initialize the ring. */ + I915_WRITE(PRB0_START, obj_priv->gtt_offset); + head = I915_READ(PRB0_HEAD) & HEAD_ADDR; + + /* G45 ring initialisation fails to reset head to zero */ + if (head != 0) { + I915_WRITE(PRB0_HEAD, 0); + DRM_DEBUG("Forced ring head to zero ctl %08x head %08x" + "tail %08x start %08x\n", I915_READ(PRB0_CTL), + I915_READ(PRB0_HEAD), I915_READ(PRB0_TAIL), + I915_READ(PRB0_START)); + } + + I915_WRITE(PRB0_CTL, ((obj->size - 4096) & RING_NR_PAGES) | + RING_NO_REPORT | RING_VALID); + + head = I915_READ(PRB0_HEAD) & HEAD_ADDR; + /* If ring head still != 0, the ring is dead */ + if (head != 0) { + DRM_ERROR("Ring initialisation failed: ctl %08x head %08x" + "tail %08x start %08x\n", I915_READ(PRB0_CTL), + I915_READ(PRB0_HEAD), I915_READ(PRB0_TAIL), + I915_READ(PRB0_START)); + return (EIO); + } + + /* Update our cache of the ring state */ + inteldrm_update_ring(dev_priv); + + return (0); +} + +void +i915_gem_cleanup_ringbuffer(struct drm_i915_private *dev_priv) +{ + if (dev_priv->ring.ring_obj == NULL) + return; + + i915_gem_object_unpin(dev_priv->ring.ring_obj); + drm_gem_object_unreference(dev_priv->ring.ring_obj); + dev_priv->ring.ring_obj = NULL; + memset(&dev_priv->ring, 0, sizeof(dev_priv->ring)); + + i915_gem_cleanup_hws(dev_priv); +} + +int +i915_gem_entervt_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + drm_i915_private_t *dev_priv = dev->dev_private; + int ret; + + if (dev_priv->mm.wedged) { + DRM_ERROR("Reenabling wedged hardware, good luck\n"); + dev_priv->mm.wedged = 0; + } + + + DRM_LOCK(); + dev_priv->mm.suspended = 0; + + ret = i915_gem_init_ringbuffer(dev_priv); + if (ret != 0) { + DRM_UNLOCK(); + return (ret); + } + + /* gtt mapping means that the inactive list may not be empty */ + KASSERT(TAILQ_EMPTY(&dev_priv->mm.active_list)); + KASSERT(TAILQ_EMPTY(&dev_priv->mm.flushing_list)); + KASSERT(TAILQ_EMPTY(&dev_priv->mm.request_list)); + DRM_UNLOCK(); + + drm_irq_install(dev); + + return (0); +} + +int +i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + int ret; + + ret = i915_gem_idle(dev_priv); + drm_irq_uninstall(dev); + return (ret); +} + +void +inteldrm_timeout(void *arg) +{ + drm_i915_private_t *dev_priv = arg; + + if (workq_add_task(dev_priv->workq, 0, i915_gem_retire_work_handler, + dev_priv, NULL) == ENOMEM) + DRM_ERROR("failed to run retire handler\n"); +} + +/* + * handle hung hardware, or error interrupts. for now print debug info. + */ +void +inteldrm_error(struct drm_i915_private *dev_priv) +{ + u_int32_t eir, ipeir, pgtbl_err, pipea_stats, pipeb_stats; + u_int8_t reset = GDRST_RENDER; + + eir = I915_READ(EIR); + pipea_stats = I915_READ(PIPEASTAT); + pipeb_stats = I915_READ(PIPEBSTAT); + + /* + * only actually check the error bits if we register one. + * else we just hung, stay silent. + */ + if (eir != 0) { + printf("render error detected, EIR: 0x%08x\n", eir); + if (IS_G4X(dev_priv)) { + if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) { + ipeir = I915_READ(IPEIR_I965); + + printf(" IPEIR: 0x%08x\n", + I915_READ(IPEIR_I965)); + printf(" IPEHR: 0x%08x\n", + I915_READ(IPEHR_I965)); + printf(" INSTDONE: 0x%08x\n", + I915_READ(INSTDONE_I965)); + printf(" INSTPS: 0x%08x\n", + I915_READ(INSTPS)); + printf(" INSTDONE1: 0x%08x\n", + I915_READ(INSTDONE1)); + printf(" ACTHD: 0x%08x\n", + I915_READ(ACTHD_I965)); + I915_WRITE(IPEIR_I965, ipeir); + (void)I915_READ(IPEIR_I965); + } + if (eir & GM45_ERROR_PAGE_TABLE) { + pgtbl_err = I915_READ(PGTBL_ER); + printf("page table error\n"); + printf(" PGTBL_ER: 0x%08x\n", pgtbl_err); + I915_WRITE(PGTBL_ER, pgtbl_err); + (void)I915_READ(PGTBL_ER); + dev_priv->mm.wedged = 1; + reset = GDRST_FULL; + + } + } else if (IS_I9XX(dev_priv) && eir & I915_ERROR_PAGE_TABLE) { + pgtbl_err = I915_READ(PGTBL_ER); + printf("page table error\n"); + printf(" PGTBL_ER: 0x%08x\n", pgtbl_err); + I915_WRITE(PGTBL_ER, pgtbl_err); + (void)I915_READ(PGTBL_ER); + dev_priv->mm.wedged = 1; + reset = GDRST_FULL; + } + if (eir & I915_ERROR_MEMORY_REFRESH) { + printf("memory refresh error\n"); + printf("PIPEASTAT: 0x%08x\n", + pipea_stats); + printf("PIPEBSTAT: 0x%08x\n", + pipeb_stats); + /* pipestat has already been acked */ + } + if (eir & I915_ERROR_INSTRUCTION) { + printf("instruction error\n"); + printf(" INSTPM: 0x%08x\n", + I915_READ(INSTPM)); + if (!IS_I965G(dev_priv)) { + ipeir = I915_READ(IPEIR); + + printf(" IPEIR: 0x%08x\n", + I915_READ(IPEIR)); + printf(" IPEHR: 0x%08x\n", + I915_READ(IPEHR)); + printf(" INSTDONE: 0x%08x\n", + I915_READ(INSTDONE)); + printf(" ACTHD: 0x%08x\n", + I915_READ(ACTHD)); + I915_WRITE(IPEIR, ipeir); + (void)I915_READ(IPEIR); + } else { + ipeir = I915_READ(IPEIR_I965); + + printf(" IPEIR: 0x%08x\n", + I915_READ(IPEIR_I965)); + printf(" IPEHR: 0x%08x\n", + I915_READ(IPEHR_I965)); + printf(" INSTDONE: 0x%08x\n", + I915_READ(INSTDONE_I965)); + printf(" INSTPS: 0x%08x\n", + I915_READ(INSTPS)); + printf(" INSTDONE1: 0x%08x\n", + I915_READ(INSTDONE1)); + printf(" ACTHD: 0x%08x\n", + I915_READ(ACTHD_I965)); + I915_WRITE(IPEIR_I965, ipeir); + (void)I915_READ(IPEIR_I965); + } + } + + I915_WRITE(EIR, eir); + eir = I915_READ(EIR); + } + /* + * nasty errors don't clear and need a reset, mask them until we reset + * else we'll get infinite interrupt storms. + */ + if (eir) { + /* print so we know that we may want to reset here too */ + if (dev_priv->mm.wedged == 0) + DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir); + I915_WRITE(EMR, I915_READ(EMR) | eir); + I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT); + } + /* + * if it was a pagetable error, or we were called from hangcheck, then + * reset the gpu. + */ + if (dev_priv->mm.wedged && workq_add_task(dev_priv->workq, 0, + inteldrm_hung, dev_priv, (void *)(uintptr_t)reset) == ENOMEM) + DRM_INFO("failed to schedule reset task\n"); + +} + +void +inteldrm_hung(void *arg, void *reset_type) +{ + struct drm_i915_private *dev_priv = arg; + struct drm_device *dev = (struct drm_device *)dev_priv->drmdev; + struct inteldrm_obj *obj_priv; + u_int8_t reset = (u_int8_t)(uintptr_t)reset_type; + + DRM_LOCK(); + if (HAS_RESET(dev_priv)) { + DRM_INFO("resetting gpu: "); + inteldrm_965_reset(dev_priv, reset); + printf("done!\n"); + } else + printf("no reset function for chipset.\n"); + + /* + * Clear out all of the requests and make everything inactive. + */ + i915_gem_retire_requests(dev_priv); + + /* + * Clear the active and flushing lists to inactive. Since + * we've reset the hardware then they're not going to get + * flushed or completed otherwise. nuke the domains since + * they're now irrelavent. + */ + while ((obj_priv = TAILQ_FIRST(&dev_priv->mm.active_list)) != NULL) { + if (obj_priv->obj.write_domain & I915_GEM_GPU_DOMAINS) { + TAILQ_REMOVE(&dev_priv->mm.gpu_write_list, + obj_priv, write_list); + atomic_clearbits_int(&obj_priv->io_flags, + I915_GPU_WRITE); + obj_priv->obj.write_domain &= ~I915_GEM_GPU_DOMAINS; + } + i915_gem_object_move_to_inactive(&obj_priv->obj);; + } + + while ((obj_priv = TAILQ_FIRST(&dev_priv->mm.flushing_list)) != NULL) { + if (obj_priv->obj.write_domain & I915_GEM_GPU_DOMAINS) { + TAILQ_REMOVE(&dev_priv->mm.gpu_write_list, + obj_priv, write_list); + atomic_clearbits_int(&obj_priv->io_flags, + I915_GPU_WRITE); + obj_priv->obj.write_domain &= ~I915_GEM_GPU_DOMAINS; + } + i915_gem_object_move_to_inactive(&obj_priv->obj); + } + + /* unbind everything */ + (void)i915_gem_evict_inactive(dev_priv); + + if (HAS_RESET(dev_priv)) + dev_priv->mm.wedged = 0; + DRM_UNLOCK(); +} + +void +inteldrm_hangcheck(void *arg) +{ + struct drm_i915_private *dev_priv = arg; + u_int32_t acthd; + + /* are we idle? */ + if (TAILQ_EMPTY(&dev_priv->mm.request_list) || + i915_seqno_passed(i915_get_gem_seqno(dev_priv), + TAILQ_LAST(&dev_priv->mm.request_list, i915_request)->seqno)) { + dev_priv->mm.hang_cnt = 0; + return; + } + + if (IS_I965G(dev_priv)) + acthd = I915_READ(ACTHD_I965); + else + acthd = I915_READ(ACTHD); + + /* if we've hit ourselves before and the hardware hasn't moved, hung. */ + if (dev_priv->mm.last_acthd == acthd) { + /* if that's twice we didn't hit it, then we're hung */ + if (++dev_priv->mm.hang_cnt >= 2) { + dev_priv->mm.hang_cnt = 0; + /* XXX atomic */ + dev_priv->mm.wedged = 1; + DRM_INFO("gpu hung!\n"); + /* XXX locking */ + wakeup(dev_priv); + inteldrm_error(dev_priv); + return; + } + } else { + dev_priv->mm.hang_cnt = 0; + } + + dev_priv->mm.last_acthd = acthd; + /* Set ourselves up again, in case we haven't added another batch */ + timeout_add_msec(&dev_priv->mm.hang_timer, 750); +} + +void +i915_move_to_tail(struct inteldrm_obj *obj_priv, struct i915_gem_list *head) +{ + i915_list_remove(obj_priv); + TAILQ_INSERT_TAIL(head, obj_priv, list); + obj_priv->current_list = head; +} + +void +i915_list_remove(struct inteldrm_obj *obj_priv) +{ + if (obj_priv->current_list != NULL) + TAILQ_REMOVE(obj_priv->current_list, obj_priv, list); + obj_priv->current_list = NULL; +} + +/* + * + * Support for managing tiling state of buffer objects. + * + * The idea behind tiling is to increase cache hit rates by rearranging + * pixel data so that a group of pixel accesses are in the same cacheline. + * Performance improvement from doing this on the back/depth buffer are on + * the order of 30%. + * + * Intel architectures make this somewhat more complicated, though, by + * adjustments made to addressing of data when the memory is in interleaved + * mode (matched pairs of DIMMS) to improve memory bandwidth. + * For interleaved memory, the CPU sends every sequential 64 bytes + * to an alternate memory channel so it can get the bandwidth from both. + * + * The GPU also rearranges its accesses for increased bandwidth to interleaved + * memory, and it matches what the CPU does for non-tiled. However, when tiled + * it does it a little differently, since one walks addresses not just in the + * X direction but also Y. So, along with alternating channels when bit + * 6 of the address flips, it also alternates when other bits flip -- Bits 9 + * (every 512 bytes, an X tile scanline) and 10 (every two X tile scanlines) + * are common to both the 915 and 965-class hardware. + * + * The CPU also sometimes XORs in higher bits as well, to improve + * bandwidth doing strided access like we do so frequently in graphics. This + * is called "Channel XOR Randomization" in the MCH documentation. The result + * is that the CPU is XORing in either bit 11 or bit 17 to bit 6 of its address + * decode. + * + * All of this bit 6 XORing has an effect on our memory management, + * as we need to make sure that the 3d driver can correctly address object + * contents. + * + * If we don't have interleaved memory, all tiling is safe and no swizzling is + * required. + * + * When bit 17 is XORed in, we simply refuse to tile at all. Bit + * 17 is not just a page offset, so as we page an object out and back in, + * individual pages in it will have different bit 17 addresses, resulting in + * each 64 bytes being swapped with its neighbor! + * + * Otherwise, if interleaved, we have to tell the 3d driver what the address + * swizzling it needs to do is, since it's writing with the CPU to the pages + * (bit 6 and potentially bit 11 XORed in), and the GPU is reading from the + * pages (bit 6, 9, and 10 XORed in), resulting in a cumulative bit swizzling + * required by the CPU of XORing in bit 6, 9, 10, and potentially 11, in order + * to match what the GPU expects. + */ + +#define MCHBAR_I915 0x44 +#define MCHBAR_I965 0x48 +#define MCHBAR_SIZE (4*4096) + +#define DEVEN_REG 0x54 +#define DEVEN_MCHBAR_EN (1 << 28) + + +/* + * Check the MCHBAR on the host bridge is enabled, and if not allocate it. + * we do not need to actually map it because we access the bar through it's + * mirror on the IGD, however, if it is disabled or not allocated then + * the mirror does not work. *sigh*. + * + * we return a trinary state: + * 0 = already enabled, or can not enable + * 1 = enabled, needs disable + * 2 = enabled, needs disable and free. + */ +int +inteldrm_setup_mchbar(struct drm_i915_private *dev_priv, + struct pci_attach_args *bpa) +{ + u_int64_t mchbar_addr; + pcireg_t tmp, low, high = 0; + u_long addr; + int reg = IS_I965G(dev_priv) ? MCHBAR_I965 : MCHBAR_I915; + int ret = 1, enabled = 0; + + if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) { + tmp = pci_conf_read(bpa->pa_pc, bpa->pa_tag, DEVEN_REG); + enabled = !!(tmp & DEVEN_MCHBAR_EN); + } else { + tmp = pci_conf_read(bpa->pa_pc, bpa->pa_tag, reg); + enabled = tmp & 1; + } + + if (enabled) { + return (0); + } + + if (IS_I965G(dev_priv)) + high = pci_conf_read(bpa->pa_pc, bpa->pa_tag, reg + 4); + low = pci_conf_read(bpa->pa_pc, bpa->pa_tag, reg); + mchbar_addr = ((u_int64_t)high << 32) | low; + + /* + * XXX need to check to see if it's allocated in the pci resources, + * right now we just check to see if there's any address there + * + * if there's no address, then we allocate one. + * note that we can't just use pci_mapreg_map here since some intel + * BARs are special in that they set bit 0 to show they're enabled, + * this is not handled by generic pci code. + */ + if (mchbar_addr == 0) { + addr = (u_long)mchbar_addr; + if (bpa->pa_memex == NULL || extent_alloc(bpa->pa_memex, + MCHBAR_SIZE, MCHBAR_SIZE, 0, 0, 0, &addr)) { + return (0); /* just say we don't need to disable */ + } else { + mchbar_addr = addr; + ret = 2; + /* We've allocated it, now fill in the BAR again */ + if (IS_I965G(dev_priv)) + pci_conf_write(bpa->pa_pc, bpa->pa_tag, + reg + 4, upper_32_bits(mchbar_addr)); + pci_conf_write(bpa->pa_pc, bpa->pa_tag, + reg, mchbar_addr & 0xffffffff); + } + } + /* set the enable bit */ + if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) { + pci_conf_write(bpa->pa_pc, bpa->pa_tag, DEVEN_REG, + tmp | DEVEN_MCHBAR_EN); + } else { + tmp = pci_conf_read(bpa->pa_pc, bpa->pa_tag, reg); + pci_conf_write(bpa->pa_pc, bpa->pa_tag, reg, tmp | 1); + } + + return (ret); +} + +/* + * we take the trinary returned from inteldrm_setup_mchbar and clean up after + * it. + */ +void +inteldrm_teardown_mchbar(struct drm_i915_private *dev_priv, + struct pci_attach_args *bpa, int disable) +{ + u_int64_t mchbar_addr; + pcireg_t tmp, low, high = 0; + int reg = IS_I965G(dev_priv) ? MCHBAR_I965 : MCHBAR_I915; + + switch(disable) { + case 2: + if (IS_I965G(dev_priv)) + high = pci_conf_read(bpa->pa_pc, bpa->pa_tag, reg + 4); + low = pci_conf_read(bpa->pa_pc, bpa->pa_tag, reg); + mchbar_addr = ((u_int64_t)high << 32) | low; + extent_free(bpa->pa_memex, mchbar_addr, MCHBAR_SIZE, 0); + /* FALLTHROUGH */ + case 1: + if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) { + tmp = pci_conf_read(bpa->pa_pc, bpa->pa_tag, DEVEN_REG); + tmp &= ~DEVEN_MCHBAR_EN; + pci_conf_write(bpa->pa_pc, bpa->pa_tag, DEVEN_REG, tmp); + } else { + tmp = pci_conf_read(bpa->pa_pc, bpa->pa_tag, reg); + tmp &= ~1; + pci_conf_write(bpa->pa_pc, bpa->pa_tag, reg, tmp); + } + break; + case 0: + default: + break; + }; +} + +/** + * Detects bit 6 swizzling of address lookup between IGD access and CPU + * access through main memory. + */ +void +inteldrm_detect_bit_6_swizzle(drm_i915_private_t *dev_priv, + struct pci_attach_args *bpa) +{ + uint32_t swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN; + uint32_t swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN; + int need_disable; + + if (!IS_I9XX(dev_priv)) { + /* As far as we know, the 865 doesn't have these bit 6 + * swizzling issues. + */ + swizzle_x = I915_BIT_6_SWIZZLE_NONE; + swizzle_y = I915_BIT_6_SWIZZLE_NONE; + } else if (IS_MOBILE(dev_priv)) { + uint32_t dcc; + + /* try to enable MCHBAR, a lot of biosen disable it */ + need_disable = inteldrm_setup_mchbar(dev_priv, bpa); + + /* On 915-945 and GM965, channel interleave by the CPU is + * determined by DCC. The CPU will alternate based on bit 6 + * in interleaved mode, and the GPU will then also alternate + * on bit 6, 9, and 10 for X, but the CPU may also optionally + * alternate based on bit 17 (XOR not disabled and XOR + * bit == 17). + */ + dcc = I915_READ(DCC); + switch (dcc & DCC_ADDRESSING_MODE_MASK) { + case DCC_ADDRESSING_MODE_SINGLE_CHANNEL: + case DCC_ADDRESSING_MODE_DUAL_CHANNEL_ASYMMETRIC: + swizzle_x = I915_BIT_6_SWIZZLE_NONE; + swizzle_y = I915_BIT_6_SWIZZLE_NONE; + break; + case DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED: + if (dcc & DCC_CHANNEL_XOR_DISABLE) { + /* This is the base swizzling by the GPU for + * tiled buffers. + */ + swizzle_x = I915_BIT_6_SWIZZLE_9_10; + swizzle_y = I915_BIT_6_SWIZZLE_9; + } else if ((dcc & DCC_CHANNEL_XOR_BIT_17) == 0) { + /* Bit 11 swizzling by the CPU in addition. */ + swizzle_x = I915_BIT_6_SWIZZLE_9_10_11; + swizzle_y = I915_BIT_6_SWIZZLE_9_11; + } else { + /* Bit 17 swizzling by the CPU in addition. */ + swizzle_x = I915_BIT_6_SWIZZLE_9_10_17; + swizzle_y = I915_BIT_6_SWIZZLE_9_17; + } + break; + } + if (dcc == 0xffffffff) { + DRM_ERROR("Couldn't read from MCHBAR. " + "Disabling tiling.\n"); + swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN; + swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN; + } + + inteldrm_teardown_mchbar(dev_priv, bpa, need_disable); + } else { + /* The 965, G33, and newer, have a very flexible memory + * configuration. It will enable dual-channel mode + * (interleaving) on as much memory as it can, and the GPU + * will additionally sometimes enable different bit 6 + * swizzling for tiled objects from the CPU. + * + * Here's what I found on G965: + * + * slot fill memory size swizzling + * 0A 0B 1A 1B 1-ch 2-ch + * 512 0 0 0 512 0 O + * 512 0 512 0 16 1008 X + * 512 0 0 512 16 1008 X + * 0 512 0 512 16 1008 X + * 1024 1024 1024 0 2048 1024 O + * + * We could probably detect this based on either the DRB + * matching, which was the case for the swizzling required in + * the table above, or from the 1-ch value being less than + * the minimum size of a rank. + */ + if (I915_READ16(C0DRB3) != I915_READ16(C1DRB3)) { + swizzle_x = I915_BIT_6_SWIZZLE_NONE; + swizzle_y = I915_BIT_6_SWIZZLE_NONE; + } else { + swizzle_x = I915_BIT_6_SWIZZLE_9_10; + swizzle_y = I915_BIT_6_SWIZZLE_9; + } + } + + dev_priv->mm.bit_6_swizzle_x = swizzle_x; + dev_priv->mm.bit_6_swizzle_y = swizzle_y; +} + +int +inteldrm_swizzle_page(struct vm_page *pg) +{ + vaddr_t va; + int i; + u_int8_t temp[64], *vaddr; + +#if defined (__HAVE_PMAP_DIRECT) + va = pmap_map_direct(pg); +#else + va = uvm_km_valloc(kernel_map, PAGE_SIZE); + if (va == 0) + return (ENOMEM); + pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg), UVM_PROT_RW); + pmap_update(pmap_kernel()); +#endif + vaddr = (u_int8_t *)va; + + for (i = 0; i < PAGE_SIZE; i += 128) { + memcpy(temp, &vaddr[i], 64); + memcpy(&vaddr[i], &vaddr[i + 64], 64); + memcpy(&vaddr[i + 64], temp, 64); + } + +#if defined (__HAVE_PMAP_DIRECT) + pmap_unmap_direct(va); +#else + pmap_kremove(va, va + PAGE_SIZE); + pmap_update(pmap_kernel()); + uvm_km_free(kernel_map, va, PAGE_SIZE); +#endif + return (0); +} + +void +i915_gem_bit_17_swizzle(struct drm_obj *obj) +{ + struct drm_device *dev = obj->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + struct vm_page *pg; + bus_dma_segment_t *segp; + int page_count = obj->size >> PAGE_SHIFT; + int i, n, ret; + + if (dev_priv->mm.bit_6_swizzle_x != I915_BIT_6_SWIZZLE_9_10_17 || + obj_priv->bit_17 == NULL) + return; + + while (i < page_count) { + /* compare bit 17 with previous one (in case we swapped). + * if they don't match we'll have to swizzle the page + */ + if ((((segp->ds_addr + n) >> 17) & 0x1) != + test_bit(i, obj_priv->bit_17)) { + /* XXX move this to somewhere where we already have pg */ + pg = PHYS_TO_VM_PAGE(segp->ds_addr + n * PAGE_SIZE); + KASSERT(pg != NULL); + ret = inteldrm_swizzle_page(pg); + if (ret) + return; + atomic_clearbits_int(&pg->pg_flags, PG_CLEAN); + } + + if (++n * PAGE_SIZE > segp->ds_len) { + n = 0; + segp++; + } + } + +} + +void +i915_gem_save_bit_17_swizzle(struct drm_obj *obj) +{ + struct drm_device *dev = obj->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + bus_dma_segment_t *segp; + int page_count = obj->size >> PAGE_SHIFT, i, n; + + if (dev_priv->mm.bit_6_swizzle_x != I915_BIT_6_SWIZZLE_9_10_17) + return; + + if (obj_priv->bit_17 == NULL) { + /* round up number of pages to a multiple of 32 so we know what + * size to make the bitmask. XXX this is wasteful with malloc + * and a better way should be done + */ + size_t nb17 = ((page_count + 31) & ~31)/32; + obj_priv->bit_17 = drm_alloc(nb17 * sizeof(u_int32_t)); + if (obj_priv-> bit_17 == NULL) { + return; + } + + } + + segp = &obj_priv->dma_segs[0]; + n = 0; + while (i < page_count) { + if ((segp->ds_addr + (n * PAGE_SIZE)) & (1 << 17)) + set_bit(i, obj_priv->bit_17); + else + clear_bit(i, obj_priv->bit_17); + + if (++n * PAGE_SIZE > segp->ds_len) { + n = 0; + segp++; + } + } +} + +bus_size_t +i915_get_fence_size(struct drm_i915_private *dev_priv, bus_size_t size) +{ + bus_size_t i, start; + + if (IS_I965G(dev_priv)) { + /* 965 can have fences anywhere, so align to gpu-page size */ + return ((size + (4096 - 1)) & ~(4096 - 1)); + } else { + /* + * Align the size to a power of two greater than the smallest + * fence size. + */ + if (IS_I9XX(dev_priv)) + start = 1024 * 1024; + else + start = 512 * 1024; + + for (i = start; i < size; i <<= 1) + ; + + return (i); + } +} + +int +i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + int tile_width; + + /* Linear is always ok */ + if (tiling_mode == I915_TILING_NONE) + return (1); + + if (!IS_I9XX(dev_priv) || (tiling_mode == I915_TILING_Y && + HAS_128_BYTE_Y_TILING(dev_priv))) + tile_width = 128; + else + tile_width = 512; + + /* Check stride and size constraints */ + if (IS_I965G(dev_priv)) { + /* fence reg has end address, so size is ok */ + if (stride / 128 > I965_FENCE_MAX_PITCH_VAL) + return (0); + } else if (IS_I9XX(dev_priv)) { + u_int32_t pitch_val = ffs(stride / tile_width) - 1; + /* + * XXX: for Y tiling, max pitch is actually 6 (8k) instead of 4 + * (2k) on the 945. + */ + if (pitch_val > I915_FENCE_MAX_PITCH_VAL || + size > (I830_FENCE_MAX_SIZE_VAL << 20)) + return (0); + } else { + u_int32_t pitch_val = ffs(stride / tile_width) - 1; + + if (pitch_val > I830_FENCE_MAX_PITCH_VAL || + size > (I830_FENCE_MAX_SIZE_VAL << 19)) + return (0); + } + + /* 965+ just needs multiples of the tile width */ + if (IS_I965G(dev_priv)) + return ((stride & (tile_width - 1)) == 0); + + + /* Pre-965 needs power-of-two */ + if (stride < tile_width || stride & (stride - 1) || + i915_get_fence_size(dev_priv, size) != size) + return (0); + return (1); +} + +int +i915_gem_object_fence_offset_ok(struct drm_obj *obj, int tiling_mode) +{ + struct drm_device *dev = obj->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj; + + if (obj_priv->dmamap == NULL || tiling_mode == I915_TILING_NONE) + return (1); + + if (!IS_I965G(dev_priv)) { + if (obj_priv->gtt_offset & (obj->size -1)) + return (0); + if (IS_I9XX(dev_priv)) { + if (obj_priv->gtt_offset & ~I915_FENCE_START_MASK) + return (0); + } else { + if (obj_priv->gtt_offset & ~I830_FENCE_START_MASK) + return (0); + } + } + return (1); +} +/** + * Sets the tiling mode of an object, returning the required swizzling of + * bit 6 of addresses in the object. + */ +int +i915_gem_set_tiling(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_i915_gem_set_tiling *args = data; + drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_obj *obj; + struct inteldrm_obj *obj_priv; + int ret = 0; + + obj = drm_gem_object_lookup(dev, file_priv, args->handle); + if (obj == NULL) + return (EBADF); + obj_priv = (struct inteldrm_obj *)obj; + + if (i915_tiling_ok(dev, args->stride, obj->size, + args->tiling_mode) == 0) { + ret = EINVAL; + DRM_LOCK(); /* for unref */ + goto out; + } + + DRM_LOCK(); + if (args->tiling_mode == I915_TILING_NONE) { + args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE; + args->stride = 0; + } else { + if (args->tiling_mode == I915_TILING_X) + args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x; + else + args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y; + /* If we can't handle the swizzling, make it untiled. */ + if (args->swizzle_mode == I915_BIT_6_SWIZZLE_UNKNOWN) { + args->tiling_mode = I915_TILING_NONE; + args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE; + args->stride = 0; + } + } + + if (args->tiling_mode != obj_priv->tiling_mode || + args->stride != obj_priv->stride) { + /* + * We need to rebind the object if its current allocation no + * longer meets the alignment restrictions for its new tiling + * mode. Otherwise we can leave it alone, but must clear any + * fence register. + */ + if (i915_gem_object_fence_offset_ok(obj, args->tiling_mode)) { + if (obj_priv->pin_count) + ret = EINVAL; + else + ret = i915_gem_object_unbind(obj, 1); + } else if (obj_priv->fence_reg != I915_FENCE_REG_NONE) { + ret = i915_gem_object_put_fence_reg(obj, 1); + } else { + inteldrm_wipe_mappings(obj); + } + if (ret != 0) { + args->tiling_mode = obj_priv->tiling_mode; + args->stride = obj_priv->stride; + goto out; + } + obj_priv->tiling_mode = args->tiling_mode; + obj_priv->stride = args->stride; + } + +out: + drm_gem_object_unreference(obj); + DRM_UNLOCK(); + + return (ret); +} + +/** + * Returns the current tiling mode and required bit 6 swizzling for the object. + */ +int +i915_gem_get_tiling(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_i915_gem_get_tiling *args = data; + drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_obj *obj; + struct inteldrm_obj *obj_priv; + + obj = drm_gem_object_lookup(dev, file_priv, args->handle); + if (obj == NULL) + return (EBADF); + obj_priv = (struct inteldrm_obj *)obj; + + DRM_LOCK(); + + args->tiling_mode = obj_priv->tiling_mode; + switch (obj_priv->tiling_mode) { + case I915_TILING_X: + args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x; + break; + case I915_TILING_Y: + args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y; + break; + case I915_TILING_NONE: + args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE; + break; + default: + DRM_ERROR("unknown tiling mode\n"); + } + + drm_gem_object_unreference(obj); + DRM_UNLOCK(); + + return 0; +} + +#endif /* INTELDRM_GEM */ + /** * inteldrm_pipe_enabled - check if a pipe is enabled * @dev: DRM device @@ -1093,3 +5247,296 @@ inteldrm_restore_state(struct drm_i915_private *dev_priv) return 0; } + +#ifdef INTELDRM_GEM +/* + * Reset the chip after a hang (965 only) + * + * The procedure that should be followed is relatively simple: + * - reset the chip using the reset reg + * - re-init context state + * - re-init Hardware status page + * - re-init ringbuffer + * - re-init interrupt state + * - re-init display + */ +void +inteldrm_965_reset(struct drm_i915_private *dev_priv, u_int8_t flags) +{ + pcireg_t reg; + int i = 0; + + if (flags == GDRST_FULL) + inteldrm_save_display(dev_priv); + + reg = pci_conf_read(dev_priv->pc, dev_priv->tag, GDRST); + /* + * Set the domains we want to reset, then bit 0 (reset itself). + * then we wait for the hardware to clear it. + */ + pci_conf_write(dev_priv->pc, dev_priv->tag, GDRST, + reg | (u_int32_t)flags | ((flags == GDRST_FULL) ? 0x1 : 0x0)); + delay(50); + /* don't clobber the rest of the register */ + pci_conf_write(dev_priv->pc, dev_priv->tag, GDRST, reg & 0xfe); + + /* if this fails we're pretty much fucked, but don't loop forever */ + do { + delay(100); + reg = pci_conf_read(dev_priv->pc, dev_priv->tag, GDRST); + } while ((reg & 0x1) && ++i < 10); + + if (reg & 0x1) + printf("bit 0 not cleared .. "); + + /* put everything back together again */ + + /* + * GTT is already up (we didn't do a pci-level reset, thank god. + * + * We don't have to restore the contexts (we don't use them yet). + * So, if X is running we need to put the ringbuffer back first. + */ + if (dev_priv->mm.suspended == 0) { + struct drm_device *dev = (struct drm_device *)dev_priv->drmdev; + if (inteldrm_start_ring(dev_priv) != 0) + panic("can't restart ring, we're fucked"); + + /* put the hardware status page back */ + if (I915_NEED_GFX_HWS(dev_priv)) + I915_WRITE(HWS_PGA, ((struct inteldrm_obj *) + dev_priv->hws_obj)->gtt_offset); + else + I915_WRITE(HWS_PGA, + dev_priv->hws_dmamem->map->dm_segs[0].ds_addr); + I915_READ(HWS_PGA); /* posting read */ + + /* so we remove the handler and can put it back in */ + DRM_UNLOCK(); + drm_irq_uninstall(dev); + drm_irq_install(dev); + DRM_LOCK(); + } else + printf("not restarting ring...\n"); + + + if (flags == GDRST_FULL) + inteldrm_restore_display(dev_priv); +} +#endif /* INTELDRM_GEM */ + +/* + * Debug code from here. + */ +#ifdef WATCH_INACTIVE +void +inteldrm_verify_inactive(struct drm_i915_private *dev_priv, char *file, + int line) +{ + struct drm_obj *obj; + struct inteldrm_obj *obj_priv; + + TAILQ_FOREACH(obj_priv, &dev_priv->mm.inactive_list, list) { + obj = (struct drm_obj *)obj_priv; + if (obj_priv->pin_count || inteldrm_is_active(obj_priv) || + obj->write_domain & I915_GEM_GPU_DOMAINS) + DRM_ERROR("inactive %p (p $d a $d w $x) %s:%d\n", + obj, obj_priv->pin_count, + inteldrm_is_active(obj_priv), + obj->write_domain, file, line); + } +} +#endif /* WATCH_INACTIVE */ + +#if (INTELDRM_DEBUG > 1) + +static const char *get_pin_flag(struct inteldrm_obj *obj_priv) +{ + if (obj_priv->pin_count > 0) + return "p"; + else + return " "; +} + +static const char *get_tiling_flag(struct inteldrm_obj *obj_priv) +{ + switch (obj_priv->tiling_mode) { + default: + case I915_TILING_NONE: return " "; + case I915_TILING_X: return "X"; + case I915_TILING_Y: return "Y"; + } +} + +void +i915_gem_seqno_info(int kdev) +{ + struct drm_device *dev = drm_get_device_from_kdev(kdev); + drm_i915_private_t *dev_priv = dev->dev_private; + + if (dev_priv->hw_status_page != NULL) { + printf("Current sequence: %d\n", i915_get_gem_seqno(dev_priv)); + } else { + printf("Current sequence: hws uninitialized\n"); + } +} + + +void +i915_interrupt_info(int kdev) +{ + struct drm_device *dev = drm_get_device_from_kdev(kdev); + drm_i915_private_t *dev_priv = dev->dev_private; + + printf("Interrupt enable: %08x\n", + I915_READ(IER)); + printf("Interrupt identity: %08x\n", + I915_READ(IIR)); + printf("Interrupt mask: %08x\n", + I915_READ(IMR)); + printf("Pipe A stat: %08x\n", + I915_READ(PIPEASTAT)); + printf("Pipe B stat: %08x\n", + I915_READ(PIPEBSTAT)); + printf("Interrupts received: 0\n"); + if (dev_priv->hw_status_page != NULL) { + printf("Current sequence: %d\n", + i915_get_gem_seqno(dev_priv)); + } else { + printf("Current sequence: hws uninitialized\n"); + } +} + +void +i915_gem_fence_regs_info(int kdev) +{ + struct drm_device *dev = drm_get_device_from_kdev(kdev); + drm_i915_private_t *dev_priv = dev->dev_private; + int i; + + printf("Reserved fences = %d\n", dev_priv->fence_reg_start); + printf("Total fences = %d\n", dev_priv->num_fence_regs); + for (i = 0; i < dev_priv->num_fence_regs; i++) { + struct drm_obj *obj = dev_priv->fence_regs[i].obj; + + if (obj == NULL) { + printf("Fenced object[%2d] = unused\n", i); + } else { + struct inteldrm_obj *obj_priv; + + obj_priv = (struct inteldrm_obj *)obj; + printf("Fenced object[%2d] = %p: %s " + "%08x %08zx %08x %s %08x %08x %d", + i, obj, get_pin_flag(obj_priv), + obj_priv->gtt_offset, + obj->size, obj_priv->stride, + get_tiling_flag(obj_priv), + obj->read_domains, obj->write_domain, + obj_priv->last_rendering_seqno); + if (obj->name) + printf(" (name: %d)", obj->name); + printf("\n"); + } + } +} + +void +i915_hws_info(int kdev) +{ + struct drm_device *dev = drm_get_device_from_kdev(kdev); + drm_i915_private_t *dev_priv = dev->dev_private; + int i; + volatile u32 *hws; + + hws = (volatile u32 *)dev_priv->hw_status_page; + if (hws == NULL) + return; + + for (i = 0; i < 4096 / sizeof(u32) / 4; i += 4) { + printf("0x%08x: 0x%08x 0x%08x 0x%08x 0x%08x\n", + i * 4, + hws[i], hws[i + 1], hws[i + 2], hws[i + 3]); + } +} + +static void +i915_dump_pages(bus_space_tag_t bst, bus_space_handle_t bsh, + bus_size_t size) +{ + bus_addr_t offset = 0; + int i = 0; + + /* + * this is a bit odd so i don't have to play with the intel + * tools too much. + */ + for (offset = 0; offset < size; offset += 4, i += 4) { + if (i == PAGE_SIZE) + i = 0; + printf("%08x : %08x\n", i, bus_space_read_4(bst, bsh, + offset)); + } +} + +void +i915_batchbuffer_info(int kdev) +{ + struct drm_device *dev = drm_get_device_from_kdev(kdev); + drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_obj *obj; + struct inteldrm_obj *obj_priv; + bus_space_handle_t bsh; + int ret; + + TAILQ_FOREACH(obj_priv, &dev_priv->mm.active_list, list) { + obj = &obj_priv->obj; + if (obj->read_domains & I915_GEM_DOMAIN_COMMAND) { + if ((ret = bus_space_subregion(dev_priv->bst, + dev_priv->aperture_bsh, obj_priv->gtt_offset, + obj->size, &bsh)) != 0) { + DRM_ERROR("Failed to map pages: %d\n", ret); + return; + } + printf("--- gtt_offset = 0x%08x\n", + obj_priv->gtt_offset); + i915_dump_pages(dev_priv->bst, bsh, obj->size); + } + } +} + +void +i915_ringbuffer_data(int kdev) +{ + struct drm_device *dev = drm_get_device_from_kdev(kdev); + drm_i915_private_t *dev_priv = dev->dev_private; + bus_size_t off; + + if (!dev_priv->ring.ring_obj) { + printf("No ringbuffer setup\n"); + return; + } + + for (off = 0; off < dev_priv->ring.size; off += 4) + printf("%08x : %08x\n", off, bus_space_read_4(dev_priv->bst, + dev_priv->ring.bsh, off)); +} + +void +i915_ringbuffer_info(int kdev) +{ + struct drm_device *dev = drm_get_device_from_kdev(kdev); + drm_i915_private_t *dev_priv = dev->dev_private; + u_int32_t head, tail; + + head = I915_READ(PRB0_HEAD) & HEAD_ADDR; + tail = I915_READ(PRB0_TAIL) & TAIL_ADDR; + + printf("RingHead : %08x\n", head); + printf("RingTail : %08x\n", tail); + printf("RingMask : %08x\n", dev_priv->ring.size - 1); + printf("RingSize : %08lx\n", dev_priv->ring.size); + printf("Acthd : %08x\n", I915_READ(IS_I965G(dev_priv) ? + ACTHD_I965 : ACTHD)); +} + +#endif diff --git a/sys/dev/pci/drm/i915_drv.h b/sys/dev/pci/drm/i915_drv.h index 3f52e270d5f..dece67d1c78 100644 --- a/sys/dev/pci/drm/i915_drv.h +++ b/sys/dev/pci/drm/i915_drv.h @@ -37,7 +37,7 @@ #define DRIVER_NAME "i915" #define DRIVER_DESC "Intel Graphics" -#define DRIVER_DATE "20080312" +#define DRIVER_DATE "20080730" enum pipe { PIPE_A = 0, @@ -59,6 +59,7 @@ enum pipe { #define DRIVER_PATCHLEVEL 0 struct inteldrm_ring { + struct drm_obj *ring_obj; bus_space_handle_t bsh; bus_size_t size; u_int32_t head; @@ -67,11 +68,21 @@ struct inteldrm_ring { u_int32_t woffset; }; +#define I915_FENCE_REG_NONE -1 + +struct inteldrm_fence { + TAILQ_ENTRY(inteldrm_fence) list; + struct drm_obj *obj; + u_int32_t last_rendering_seqno; +}; + typedef struct drm_i915_private { struct device dev; struct device *drmdev; + bus_dma_tag_t agpdmat; /* tag from intagp for GEM */ bus_dma_tag_t dmat; bus_space_tag_t bst; + bus_space_handle_t aperture_bsh; u_long flags; u_int16_t pci_device; @@ -86,8 +97,21 @@ typedef struct drm_i915_private { struct drm_local_map *sarea; drm_i915_sarea_t *sarea_priv; + union flush { + struct { + bus_space_tag_t bst; + bus_space_handle_t bsh; + } i9xx; + struct { + bus_dma_segment_t seg; + caddr_t kva; + } i8xx; + } ifp; struct inteldrm_ring ring; + struct workq *workq; + struct vm_page *pgs; struct drm_local_map hws_map; + struct drm_obj *hws_obj; struct drm_dmamem *hws_dmamem; void *hw_status_page; unsigned int status_gfx_addr; @@ -103,6 +127,10 @@ typedef struct drm_i915_private { int allow_batchbuffer; + struct inteldrm_fence fence_regs[16]; /* 965 */ + int fence_reg_start; /* 4 by default */ + int num_fence_regs; /* 8 pre-965, 16 post */ + /* Register state */ u8 saveLBB; u32 saveDSPACNTR; @@ -202,8 +230,107 @@ typedef struct drm_i915_private { u32 saveCURBPOS; u32 saveCURBBASE; u32 saveCURSIZE; + + struct { + /** + * List of objects currently involved in rendering from the + * ringbuffer. + * + * Includes buffers having the contents of their GPU caches + * flushed, not necessarily primitives. last_rendering_seqno + * represents when the rendering involved will be completed. + * + * A reference is held on the buffer while on this list. + */ + TAILQ_HEAD(i915_gem_list, inteldrm_obj) active_list; + + /** + * List of objects which are not in the ringbuffer but which + * still have a write_domain which needs to be flushed before + * unbinding. + * + * last_rendering_seqno is 0 while an object is in this list + * + * A reference is held on the buffer while on this list. + */ + struct i915_gem_list flushing_list; + + /* + * list of objects currently pending a GPU write flush. + * + * All elements on this list will either be on the active + * or flushing list, last rendiering_seqno differentiates the + * two. + */ + struct i915_gem_list gpu_write_list; + /** + * LRU list of objects which are not in the ringbuffer and + * are ready to unbind, but are still in the GTT. + * + * last_rendering_seqno is 0 while an object is in this list + * + * A reference is not held on the buffer while on this list, + * as merely being GTT-bound shouldn't prevent its being + * freed, and we'll pull it off the list in the free path. + */ + struct i915_gem_list inactive_list; + + /* Fence LRU */ + TAILQ_HEAD(i915_fence, inteldrm_fence) fence_list; + + /** + * List of breadcrumbs associated with GPU requests currently + * outstanding. + */ + TAILQ_HEAD(i915_request , inteldrm_request) request_list; + + /** + * We leave the user IRQ off as much as possible, + * but this means that requests will finish and never + * be retired once the system goes idle. Set a timer to + * fire periodically while the ring is running. When it + * fires, go retire requests in a workq. + */ + struct timeout retire_timer; + struct timeout hang_timer; + /* for hangcheck */ + int hang_cnt; + u_int32_t last_acthd; + + uint32_t next_gem_seqno; + + /** + * Flag if the X Server, and thus DRM, is not currently in + * control of the device. + * + * This is set between LeaveVT and EnterVT. It needs to be + * replaced with a semaphore. It also needs to be + * transitioned away from for kernel modesetting. + */ + int suspended; + + /** + * Flag if the hardware appears to be wedged. + * + * This is set when attempts to idle the device timeout. + * It prevents command submission from occuring and makes + * every pending request fail + */ + int wedged; + + /** Bit 6 swizzling required for X tiling */ + uint32_t bit_6_swizzle_x; + /** Bit 6 swizzling required for Y tiling */ + uint32_t bit_6_swizzle_y; + } mm; } drm_i915_private_t; +struct inteldrm_file { + struct drm_file file_priv; + struct { + } mm; +}; + /* chip type flags */ #define CHIP_I830 0x0001 #define CHIP_I845G 0x0002 @@ -222,18 +349,86 @@ typedef struct drm_i915_private { #define CHIP_M 0x4000 #define CHIP_HWS 0x8000 +/** driver private structure attached to each drm_gem_object */ +struct inteldrm_obj { + struct drm_obj obj; + + /** This object's place on the active/flushing/inactive lists */ + TAILQ_ENTRY(inteldrm_obj) list; + TAILQ_ENTRY(inteldrm_obj) write_list; + struct i915_gem_list *current_list; + /* GTT binding. */ + bus_dmamap_t dmamap; + bus_dma_segment_t *dma_segs; + /* Current offset of the object in GTT space. */ + bus_addr_t gtt_offset; + u_int32_t *bit_17; + /* + * This is set if the object is on the active or flushing lists + * (has pending rendering), and is not set if it's on inactive (ready + * to be unbound). + */ +#define I915_ACTIVE 0x0001 /* being used by the gpu. */ +#define I915_IN_EXEC 0x0002 /* being processed in execbuffer */ +#define I915_USER_PINNED 0x0004 /* BO has been pinned from userland */ +#define I915_GPU_WRITE 0x0008 /* BO has been not flushed */ +#define I915_DONTNEED 0x0010 /* BO backing pages purgable */ +#define I915_PURGED 0x0020 /* BO backing pages purged */ +#define I915_DIRTY 0x0040 /* BO written to since last bound */ +#define I915_EXEC_NEEDS_FENCE 0x0080 /* being processed but will need fence*/ +#define I915_FENCED_EXEC 0x0100 /* Most recent exec needs fence */ + int io_flags; + /* extra flags to bus_dma */ + int dma_flags; + /* Fence register for this object. needed for tiling. */ + int fence_reg; + /** refcount for times pinned this object in GTT space */ + int pin_count; + /* number of times pinned by pin ioctl. */ + u_int user_pin_count; + + /** Breadcrumb of last rendering to the buffer. */ + u_int32_t last_rendering_seqno; + /** Current tiling mode for the object. */ + u_int32_t tiling_mode; + u_int32_t stride; +}; + +#define inteldrm_is_active(obj_priv) (obj_priv->io_flags & I915_ACTIVE) +#define inteldrm_is_dirty(obj_priv) (obj_priv->io_flags & I915_DIRTY) +#define inteldrm_exec_needs_fence(obj_priv) \ + (obj_priv->io_flags & I915_EXEC_NEEDS_FENCE) +#define inteldrm_needs_fence(obj_priv) (obj_priv->io_flags & I915_FENCED_EXEC) + +/** + * Request queue structure. + * + * The request queue allows us to note sequence numbers that have been emitted + * and may be associated with active buffers to be retired. + * + * By keeping this list, we can avoid having to do questionable + * sequence-number comparisons on buffer last_rendering_seqnos, and associate + * an emission time with seqnos for tracking how far ahead of the GPU we are. + */ +struct inteldrm_request { + TAILQ_ENTRY(inteldrm_request) list; + /** GEM sequence number associated with this request. */ + uint32_t seqno; +}; + u_int32_t inteldrm_read_hws(struct drm_i915_private *, int); int inteldrm_wait_ring(struct drm_i915_private *dev, int n); void inteldrm_begin_ring(struct drm_i915_private *, int); void inteldrm_out_ring(struct drm_i915_private *, u_int32_t); void inteldrm_advance_ring(struct drm_i915_private *); void inteldrm_update_ring(struct drm_i915_private *); +void inteldrm_error(struct drm_i915_private *); int inteldrm_pipe_enabled(struct drm_i915_private *, int); /* i915_dma.c */ extern void i915_emit_breadcrumb(struct drm_device *dev); -extern int i915_dispatch_batchbuffer(struct drm_device * dev, - drm_i915_batchbuffer_t * batch); +void i915_emit_box(struct drm_device * dev, struct drm_clip_rect * boxes, + int DR1, int DR4); int i915_dma_cleanup(struct drm_device *); int i915_init_phys_hws(drm_i915_private_t *, bus_dma_tag_t); @@ -250,6 +445,7 @@ extern void i915_driver_irq_uninstall(struct drm_device * dev); extern int i915_vblank_pipe_get(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int i915_emit_irq(struct drm_device * dev); +extern int i915_wait_irq(struct drm_device * dev, int irq_nr); extern int i915_enable_vblank(struct drm_device *dev, int crtc); extern void i915_disable_vblank(struct drm_device *dev, int crtc); extern u32 i915_get_vblank_counter(struct drm_device *dev, int crtc); @@ -305,6 +501,11 @@ read64(struct drm_i915_private *dev_priv, bus_size_t off) #define I915_WRITE8(reg,val) bus_space_write_1(dev_priv->regs->bst, \ dev_priv->regs->bsh, (reg), (val)) +#define RING_LOCK_TEST_WITH_RETURN(dev, file_priv) do { \ + if (((drm_i915_private_t *)dev->dev_private)->ring.ring_obj == NULL) \ + LOCK_TEST_WITH_RETURN(dev, file_priv); \ +} while (0) + #define INTELDRM_VERBOSE 0 #if INTELDRM_VERBOSE > 0 #define INTELDRM_VPRINTF(fmt, args...) DRM_INFO(fmt, ##args) @@ -2121,6 +2322,71 @@ read64(struct drm_i915_private *dev_priv, bus_size_t off) #define I915_NEED_GFX_HWS(dev_priv) (dev_priv->flags & CHIP_HWS) +#define HAS_RESET(dev_priv) IS_I965G(dev_priv) + +/* + * Interrupts that are always left unmasked. + * + * Since pipe events are edge-triggered from the PIPESTAT register to IIRC, + * we leave them always unmasked in IMR and then control enabling them through + * PIPESTAT alone. + */ +#define I915_INTERRUPT_ENABLE_FIX \ + (I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | \ + I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | \ + I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT) + +/* Interrupts that we mask and unmask at runtime */ +#define I915_INTERRUPT_ENABLE_VAR (I915_USER_INTERRUPT) + +/* These are all of the interrupts used by the driver */ +#define I915_INTERRUPT_ENABLE_MASK \ + (I915_INTERRUPT_ENABLE_FIX | \ + I915_INTERRUPT_ENABLE_VAR) + +#define printeir(val) printf("%s: error reg: %b\n", __func__, val, \ + "\20\x10PTEERR\x2REFRESHERR\x1INSTERR") + + +/* + * With the i45 and later, Y tiling got adjusted so that it was 32 128-byte + * rows, which changes the alignment requirements and fence programming. + */ +#define HAS_128_BYTE_Y_TILING(dev_priv) (IS_I9XX(dev_priv) && \ + !(IS_I915G(dev_priv) || IS_I915GM(dev_priv))) + #define PRIMARY_RINGBUFFER_SIZE (128*1024) +/* Inlines */ + +/** + * Returns true if seq1 is later than seq2. + */ +static __inline int +i915_seqno_passed(uint32_t seq1, uint32_t seq2) +{ + return ((int32_t)(seq1 - seq2) >= 0); +} + +/* + * Read seqence number from the Hardware status page. + */ +static __inline u_int32_t +i915_get_gem_seqno(struct drm_i915_private *dev_priv) +{ + return (READ_HWSP(dev_priv, I915_GEM_HWS_INDEX)); +} + +static __inline int +i915_obj_purgeable(struct inteldrm_obj *obj_priv) +{ + return (obj_priv->io_flags & I915_DONTNEED); +} + +static __inline int +i915_obj_purged(struct inteldrm_obj *obj_priv) +{ + return (obj_priv->io_flags & I915_PURGED); +} + #endif diff --git a/sys/dev/pci/drm/i915_irq.c b/sys/dev/pci/drm/i915_irq.c index 2fd852de163..f7a0af2921c 100644 --- a/sys/dev/pci/drm/i915_irq.c +++ b/sys/dev/pci/drm/i915_irq.c @@ -38,25 +38,6 @@ void i915_enable_pipestat(drm_i915_private_t *, int, u_int32_t); void i915_disable_pipestat(drm_i915_private_t *, int, u_int32_t); int i915_wait_irq(struct drm_device *, int); -/* - * Interrupts that are always left unmasked. - * - * Since pipe events are edge-triggered from the PIPESTAT register to IIRC, - * we leave them always unmasked in IMR and then control enabling them through - * PIPESTAT alone. - */ -#define I915_INTERRUPT_ENABLE_FIX \ - (I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | \ - I915_DISPLAY_PIPE_B_EVENT_INTERRUPT) - -/* Interrupts that we mask and unmask at runtime */ -#define I915_INTERRUPT_ENABLE_VAR (I915_USER_INTERRUPT) - -/* These are all of the interrupts used by the driver */ -#define I915_INTERRUPT_ENABLE_MASK \ - (I915_INTERRUPT_ENABLE_FIX | \ - I915_INTERRUPT_ENABLE_VAR) - inline void i915_enable_irq(drm_i915_private_t *dev_priv, u_int32_t mask) { @@ -142,55 +123,6 @@ i915_get_vblank_counter(struct drm_device *dev, int pipe) } int -inteldrm_intr(void *arg) -{ - struct drm_device *dev = arg; - drm_i915_private_t *dev_priv = dev->dev_private; - u_int32_t iir, pipea_stats = 0, pipeb_stats = 0; - - /* - * lock is to protect from writes to PIPESTAT and IMR from other cores. - */ - mtx_enter(&dev_priv->user_irq_lock); - iir = I915_READ(IIR); - if (iir == 0) { - mtx_leave(&dev_priv->user_irq_lock); - return (0); - } - - /* - * Clear the PIPE(A|B)STAT regs before the IIR - */ - if (iir & I915_DISPLAY_PIPE_A_EVENT_INTERRUPT) { - pipea_stats = I915_READ(PIPEASTAT); - I915_WRITE(PIPEASTAT, pipea_stats); - } - if (iir & I915_DISPLAY_PIPE_B_EVENT_INTERRUPT) { - pipeb_stats = I915_READ(PIPEBSTAT); - I915_WRITE(PIPEBSTAT, pipeb_stats); - } - - I915_WRITE(IIR, iir); - (void)I915_READ(IIR); /* Flush posted writes */ - - if (dev_priv->sarea_priv != NULL) - dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); - - if (iir & I915_USER_INTERRUPT) - wakeup(dev_priv); - - mtx_leave(&dev_priv->user_irq_lock); - - if (pipea_stats & I915_VBLANK_INTERRUPT_STATUS) - drm_handle_vblank(dev, 0); - - if (pipeb_stats & I915_VBLANK_INTERRUPT_STATUS) - drm_handle_vblank(dev, 1); - - return (1); -} - -int i915_emit_irq(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; @@ -212,18 +144,14 @@ i915_emit_irq(struct drm_device *dev) void i915_user_irq_get(struct drm_i915_private *dev_priv) { - struct drm_device *dev = (struct drm_device *)dev_priv->drmdev; - - if (dev->irq_enabled && (++dev_priv->user_irq_refcount == 1)) + if (++dev_priv->user_irq_refcount == 1) i915_enable_irq(dev_priv, I915_USER_INTERRUPT); } void i915_user_irq_put(struct drm_i915_private *dev_priv) { - struct drm_device *dev = (struct drm_device *)dev_priv->drmdev; - - if (dev->irq_enabled && (--dev_priv->user_irq_refcount == 0)) + if (--dev_priv->user_irq_refcount == 0) i915_disable_irq(dev_priv, I915_USER_INTERRUPT); } @@ -262,7 +190,7 @@ i915_irq_emit(struct drm_device *dev, void *data, struct drm_file *file_priv) drm_i915_irq_emit_t *emit = data; int result; - LOCK_TEST_WITH_RETURN(dev, file_priv); + RING_LOCK_TEST_WITH_RETURN(dev, file_priv); if (!dev_priv) { DRM_ERROR("called with no initialization\n"); @@ -350,23 +278,31 @@ i915_driver_irq_install(struct drm_device *dev) I915_WRITE(IER, 0x0); (void)I915_READ(IER); - dev_priv->irqh = pci_intr_establish(dev_priv->pc, dev_priv->ih, IPL_BIO, - inteldrm_intr, dev, dev_priv->dev.dv_xname); - if (dev_priv->irqh == NULL) - return (ENOENT); - dev->vblank->vb_max = 0xffffff; /* only 24 bits of frame count */ if (IS_G4X(dev_priv)) dev->vblank->vb_max = 0xffffffff; - /* Unmask the interrupts that we always want on. */ - dev_priv->irq_mask_reg = ~I915_INTERRUPT_ENABLE_FIX; - - dev_priv->pipestat[0] = dev_priv->pipestat[1] = 0; +#ifdef INTELDRM_GEM + /* + * Enable some error detection, note the instruction error mask + * bit is reserved, so we leave it masked. + */ + I915_WRITE(EMR, IS_G4X(dev_priv) ? + ~(GM45_ERROR_PAGE_TABLE | GM45_ERROR_MEM_PRIV | + GM45_ERROR_CP_PRIV | I915_ERROR_MEMORY_REFRESH) : + ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH)); +#endif /* INTELDRM_GEM */ - /* Disable pipe interrupt enables, clear pending pipe status */ - I915_WRITE(PIPEASTAT, I915_READ(PIPEASTAT) & 0x8000ffff); - I915_WRITE(PIPEBSTAT, I915_READ(PIPEBSTAT) & 0x8000ffff); + /* + * Disable pipe interrupt enables, clear pending pipe status + * add back in the enabled interrupts from previous iterations + * (say in the reset case where we want vblank interrupts etc to be + * switched back on if they were running + */ + I915_WRITE(PIPEASTAT, (I915_READ(PIPEASTAT) & 0x8000ffff) | + dev_priv->pipestat[0]); + I915_WRITE(PIPEBSTAT, (I915_READ(PIPEBSTAT) & 0x8000ffff) | + dev_priv->pipestat[1]); /* Clear pending interrupt status */ I915_WRITE(IIR, I915_READ(IIR)); @@ -382,9 +318,6 @@ i915_driver_irq_uninstall(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; - if (!dev_priv) - return; - I915_WRITE(HWSTAM, 0xffffffff); I915_WRITE(PIPEASTAT, 0); I915_WRITE(PIPEBSTAT, 0); @@ -394,6 +327,4 @@ i915_driver_irq_uninstall(struct drm_device *dev) I915_WRITE(PIPEASTAT, I915_READ(PIPEASTAT) & 0x8000ffff); I915_WRITE(PIPEBSTAT, I915_READ(PIPEBSTAT) & 0x8000ffff); I915_WRITE(IIR, I915_READ(IIR)); - - pci_intr_disestablish(dev_priv->pc, dev_priv->irqh); } |